2449 lines
100 KiB
Bash
Executable File
2449 lines
100 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# scripts/manual/run_server_to_rc_mode_matrix.sh
|
|
# Manual: validate server-generated UVC/UAC output against the RC target across
|
|
# the UVC modes the UI advertises. This is still a hardware-in-the-loop probe:
|
|
# it captures the real Tethys UVC/UAC endpoints and summarizes sync,
|
|
# freshness, and smoothness for each mode.
|
|
# Not part of CI: it needs the Theia/Tethys lab hosts and live USB gadget state.
|
|
#
|
|
# Reconfigure mode is intentionally a fast runtime path: it updates the remote
|
|
# Lesavka env files and cycles the UVC gadget, but it does not rebuild or
|
|
# reinstall the server binary for each mode.
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
|
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"
|
|
|
|
if (( EUID == 0 )); then
|
|
printf 'Do not run this matrix with local sudo.\n' >&2
|
|
printf 'Run it as your normal workstation user; the script will request sudo on the remote server host only when reconfiguring UVC.\n' >&2
|
|
exit 64
|
|
fi
|
|
|
|
TETHYS_HOST=${TETHYS_HOST:-tethys}
|
|
LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST:-theia}
|
|
LESAVKA_SERVER_CONNECT_HOST=${LESAVKA_SERVER_CONNECT_HOST:-38.28.125.112}
|
|
LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-auto}
|
|
LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https}
|
|
LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}
|
|
LESAVKA_SERVER_REPO=${LESAVKA_SERVER_REPO:-auto}
|
|
SSH_OPTS=${SSH_OPTS:-"-o BatchMode=yes -o ConnectTimeout=30"}
|
|
|
|
LESAVKA_SERVER_RC_CORE_WEBCAM_MODES=${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES:-1280x720@20,1280x720@30,1920x1080@20,1920x1080@30}
|
|
LESAVKA_SERVER_RC_MODES=${LESAVKA_SERVER_RC_MODES:-${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES}}
|
|
LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US:-${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}}
|
|
LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US:-135090}
|
|
LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US:-1280x720@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1280x720@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US}}
|
|
LESAVKA_SERVER_RC_MODE_DELAYS_US=${LESAVKA_SERVER_RC_MODE_DELAYS_US:-1280x720@20=162659,1280x720@30=135090,1920x1080@20=160045,1920x1080@30=127952}
|
|
LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES=${LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES:-1280x720,1920x1080}
|
|
LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS=${LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS:-20,30}
|
|
LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX:-Logitech|BRIO|C9[0-9]+|HD UVC WebCam|USB2[.]0 HD|Integrated Camera|Webcam|Camera}
|
|
LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX:-Lesavka|UGREEN|MACROSILICON|Composite|Capture}
|
|
LESAVKA_SERVER_RC_MODE_SOURCE=${LESAVKA_SERVER_RC_MODE_SOURCE:-configured}
|
|
LESAVKA_SERVER_RC_RECONFIGURE=${LESAVKA_SERVER_RC_RECONFIGURE:-0}
|
|
LESAVKA_SERVER_RC_RECONFIGURE_REF=${LESAVKA_SERVER_RC_RECONFIGURE_REF:-master}
|
|
LESAVKA_SERVER_RC_RECONFIGURE_UPDATE=${LESAVKA_SERVER_RC_RECONFIGURE_UPDATE:-0}
|
|
LESAVKA_SERVER_RC_RECONFIGURE_COMMAND=${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND:-}
|
|
LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY:-runtime}
|
|
LESAVKA_SERVER_RC_RECONFIGURE_CODEC=${LESAVKA_SERVER_RC_RECONFIGURE_CODEC:-mjpeg}
|
|
LESAVKA_SERVER_RC_ALLOW_GADGET_RESET=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET:-1}
|
|
LESAVKA_SERVER_RC_FORCE_GADGET_REBUILD=${LESAVKA_SERVER_RC_FORCE_GADGET_REBUILD:-1}
|
|
LESAVKA_SERVER_RC_RECONFIGURE_SETTLE_SECONDS=${LESAVKA_SERVER_RC_RECONFIGURE_SETTLE_SECONDS:-4}
|
|
LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE=${LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE:-0}
|
|
LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY=${LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY:-1}
|
|
LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD=${LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD:-}
|
|
LESAVKA_SERVER_RC_START_DELAY_SECONDS=${LESAVKA_SERVER_RC_START_DELAY_SECONDS:-0}
|
|
LESAVKA_SERVER_RC_WAIT_TETHYS_READY=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY:-1}
|
|
LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS:-60}
|
|
LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS:-6}
|
|
LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS:-3}
|
|
LESAVKA_SERVER_RC_PROBE_PREBUILD=${LESAVKA_SERVER_RC_PROBE_PREBUILD:-1}
|
|
LESAVKA_SERVER_RC_CONTINUE_ON_FAIL=${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL:-1}
|
|
LESAVKA_SERVER_RC_TUNE_DELAYS=${LESAVKA_SERVER_RC_TUNE_DELAYS:-1}
|
|
LESAVKA_SERVER_RC_TUNE_CONFIRM=${LESAVKA_SERVER_RC_TUNE_CONFIRM:-1}
|
|
LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-13}
|
|
LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS:-1000}
|
|
LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS:-80}
|
|
LESAVKA_SERVER_RC_TUNE_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US:-500000}
|
|
LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000}
|
|
LESAVKA_SERVER_RC_REPEAT_COUNT=${LESAVKA_SERVER_RC_REPEAT_COUNT:-1}
|
|
LESAVKA_SERVER_RC_VERBOSE_PROBES=${LESAVKA_SERVER_RC_VERBOSE_PROBES:-1}
|
|
LESAVKA_SERVER_RC_STATIC_MIN_RUNS=${LESAVKA_SERVER_RC_STATIC_MIN_RUNS:-3}
|
|
LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US=${LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US:-30000}
|
|
LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS=${LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS:-35}
|
|
LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS=${LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS:-20}
|
|
LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS=${LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS:-1}
|
|
LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS=${LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS:-0}
|
|
|
|
LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350}
|
|
LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS:-100}
|
|
LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}
|
|
LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS=${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS:-${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}}
|
|
LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS:-1}
|
|
LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS=${LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS:-1}
|
|
LESAVKA_SERVER_RC_MIN_CODED_PAIRS=${LESAVKA_SERVER_RC_MIN_CODED_PAIRS:-${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}}
|
|
LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS=${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS:-0}
|
|
LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS:-0}
|
|
LESAVKA_SERVER_RC_SIGNAL_READY=${LESAVKA_SERVER_RC_SIGNAL_READY:-1}
|
|
LESAVKA_SERVER_RC_SIGNAL_READY_MODE=${LESAVKA_SERVER_RC_SIGNAL_READY_MODE:-conditioned_capture}
|
|
LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS:-3}
|
|
LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS:-12}
|
|
LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS:-1}
|
|
LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS:-4}
|
|
LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS:-5}
|
|
LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS:-12}
|
|
LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS:-1}
|
|
LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS:-1}
|
|
LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES:-${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}}
|
|
LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW=${LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW:-auto}
|
|
|
|
LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS=${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS:-0}
|
|
LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS=${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS:-0}
|
|
LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS=${LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS:-1}
|
|
LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS=${LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS:-1}
|
|
LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES:-12}
|
|
LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES:-12}
|
|
LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES:-5}
|
|
LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS=${LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS:-2}
|
|
|
|
PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-20}
|
|
PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4}
|
|
PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}
|
|
REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}
|
|
REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-cfr}
|
|
REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse}
|
|
REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto}
|
|
REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}
|
|
REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS:-1}
|
|
LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}
|
|
|
|
STAMP="$(date +%Y%m%d-%H%M%S)"
|
|
LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}
|
|
MATRIX_REPORT_DIR=${MATRIX_REPORT_DIR:-"${LOCAL_OUTPUT_DIR%/}/lesavka-server-rc-mode-matrix-${STAMP}"}
|
|
MATRIX_SUMMARY_JSON="${MATRIX_REPORT_DIR}/mode-matrix-summary.json"
|
|
MATRIX_SUMMARY_CSV="${MATRIX_REPORT_DIR}/mode-matrix-summary.csv"
|
|
MATRIX_SUMMARY_TXT="${MATRIX_REPORT_DIR}/mode-matrix-summary.txt"
|
|
MATRIX_DELAY_JSON="${MATRIX_REPORT_DIR}/mode-delay-recommendations.json"
|
|
MATRIX_DELAY_ENV="${MATRIX_REPORT_DIR}/mode-delay-recommendations.env"
|
|
MATRIX_STATIC_JSON="${MATRIX_REPORT_DIR}/mode-static-calibration.json"
|
|
MATRIX_STATIC_CSV="${MATRIX_REPORT_DIR}/mode-static-calibration.csv"
|
|
MATRIX_STATIC_TXT="${MATRIX_REPORT_DIR}/mode-static-calibration.txt"
|
|
MATRIX_STATIC_ENV="${MATRIX_REPORT_DIR}/mode-static-calibration.env"
|
|
MATRIX_RUN_LOG="${MATRIX_REPORT_DIR}/mode-matrix-run.log"
|
|
mkdir -p "${MATRIX_REPORT_DIR}"
|
|
exec > >(tee -a "${MATRIX_RUN_LOG}") 2>&1
|
|
|
|
mode_id() {
|
|
local mode=$1
|
|
printf '%s\n' "${mode//@/_}" | tr -c '[:alnum:]_.-' '_'
|
|
}
|
|
|
|
parse_mode() {
|
|
local mode=$1
|
|
if [[ ! "${mode}" =~ ^([0-9]+)x([0-9]+)@([0-9]+)$ ]]; then
|
|
printf 'invalid mode %s; expected WIDTHxHEIGHT@FPS\n' "${mode}" >&2
|
|
exit 64
|
|
fi
|
|
printf '%s %s %s\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}"
|
|
}
|
|
|
|
lookup_mode_delay_us() {
|
|
local mode=$1
|
|
local delay_map=$2
|
|
local default_value=$3
|
|
local entry key value
|
|
IFS=',' read -r -a delay_entries <<<"${delay_map}"
|
|
for entry in "${delay_entries[@]}"; do
|
|
key=${entry%%=*}
|
|
value=${entry#*=}
|
|
if [[ "${key}" == "${mode}" && "${value}" =~ ^-?[0-9]+$ ]]; then
|
|
printf '%s\n' "${value}"
|
|
return 0
|
|
fi
|
|
done
|
|
printf '%s\n' "${default_value}"
|
|
}
|
|
|
|
lookup_audio_delay_us() {
|
|
lookup_mode_delay_us "$1" "${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}" "${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US}"
|
|
}
|
|
|
|
lookup_video_delay_us() {
|
|
lookup_mode_delay_us "$1" "${LESAVKA_SERVER_RC_MODE_DELAYS_US}" "${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US}"
|
|
}
|
|
|
|
artifact_dir_from_log() {
|
|
local log_path=$1
|
|
local search_dir=$2
|
|
local artifact_dir
|
|
artifact_dir="$(awk -F': ' '/^artifact_dir: / {print $2}' "${log_path}" 2>/dev/null | tail -n1 || true)"
|
|
if [[ -z "${artifact_dir}" ]]; then
|
|
artifact_dir="$(find "${search_dir}" -mindepth 1 -maxdepth 1 -type d -name 'lesavka-output-delay-probe-*' | sort | tail -n1 || true)"
|
|
fi
|
|
printf '%s\n' "${artifact_dir}"
|
|
}
|
|
|
|
RUN_MODE_PROBE_STATUS=0
|
|
run_mode_probe() {
|
|
local width=$1
|
|
local height=$2
|
|
local fps=$3
|
|
local audio_delay_us=$4
|
|
local video_delay_us=$5
|
|
local output_dir=$6
|
|
local log_path=$7
|
|
local probe_duration_seconds=${8:-${PROBE_DURATION_SECONDS}}
|
|
local probe_warmup_seconds=${9:-${PROBE_WARMUP_SECONDS}}
|
|
local min_pairs=${10:-${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}}
|
|
local conditioning_seconds=${11:-${LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS}}
|
|
local analysis_timeline_window=${LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW}
|
|
if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" == "0" ]]; then
|
|
conditioning_seconds=0
|
|
fi
|
|
if [[ "${analysis_timeline_window}" == "auto" ]]; then
|
|
if [[ "${conditioning_seconds}" =~ ^[0-9]+$ && "${conditioning_seconds}" -gt 0 ]]; then
|
|
analysis_timeline_window=1
|
|
else
|
|
analysis_timeline_window=0
|
|
fi
|
|
fi
|
|
|
|
local -a probe_env=(
|
|
"TETHYS_HOST=${TETHYS_HOST}"
|
|
"LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST}"
|
|
"LESAVKA_SERVER_CONNECT_HOST=${LESAVKA_SERVER_CONNECT_HOST}"
|
|
"LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR}"
|
|
"LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME}"
|
|
"LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN}"
|
|
"SSH_OPTS=${SSH_OPTS}"
|
|
"REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL}"
|
|
"REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE}"
|
|
"REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK}"
|
|
"REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE}"
|
|
"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}"
|
|
"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}"
|
|
"REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS}"
|
|
"PROBE_PREBUILD=0"
|
|
"VIDEO_SIZE=${width}x${height}"
|
|
"VIDEO_FPS=${fps}"
|
|
"VIDEO_FORMAT=mjpeg"
|
|
"REMOTE_EXPECT_UVC_WIDTH=${width}"
|
|
"REMOTE_EXPECT_UVC_HEIGHT=${height}"
|
|
"REMOTE_EXPECT_UVC_FPS=${fps}"
|
|
"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${audio_delay_us}"
|
|
"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${video_delay_us}"
|
|
"LESAVKA_OUTPUT_DELAY_APPLY=0"
|
|
"LESAVKA_OUTPUT_DELAY_SAVE=0"
|
|
"LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0"
|
|
"LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${min_pairs}"
|
|
"LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}"
|
|
"LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}"
|
|
"LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}"
|
|
"LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}"
|
|
"LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}"
|
|
"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}"
|
|
"LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=${min_pairs}"
|
|
"PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES}"
|
|
"PROBE_CONDITIONING_SECONDS=${conditioning_seconds}"
|
|
"PROBE_CONDITIONING_WARMUP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS}"
|
|
"PROBE_CONDITIONING_GAP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS}"
|
|
"PROBE_CONDITIONING_EVENT_WIDTH_CODES=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES}"
|
|
"PROBE_DURATION_SECONDS=${probe_duration_seconds}"
|
|
"PROBE_WARMUP_SECONDS=${probe_warmup_seconds}"
|
|
"ANALYSIS_TIMELINE_WINDOW=${analysis_timeline_window}"
|
|
"LOCAL_OUTPUT_DIR=${output_dir}"
|
|
)
|
|
|
|
set +e
|
|
if [[ "${LESAVKA_SERVER_RC_VERBOSE_PROBES}" == "0" ]]; then
|
|
env "${probe_env[@]}" "${SCRIPT_DIR}/run_upstream_av_sync.sh" >"${log_path}" 2>&1
|
|
RUN_MODE_PROBE_STATUS=$?
|
|
else
|
|
env "${probe_env[@]}" "${SCRIPT_DIR}/run_upstream_av_sync.sh" 2>&1 | tee "${log_path}"
|
|
RUN_MODE_PROBE_STATUS=${PIPESTATUS[0]}
|
|
fi
|
|
set -e
|
|
}
|
|
|
|
write_tune_candidate_env() {
|
|
local mode_result=$1
|
|
local output_env=$2
|
|
python3 - <<'PY' \
|
|
"${mode_result}" \
|
|
"${output_env}" \
|
|
"${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}" \
|
|
"${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}" \
|
|
"${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" \
|
|
"${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}" \
|
|
"${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}"
|
|
import json
|
|
import math
|
|
import pathlib
|
|
import shlex
|
|
import sys
|
|
|
|
(
|
|
result_path,
|
|
output_env_path,
|
|
min_pairs_raw,
|
|
max_abs_skew_raw,
|
|
max_drift_raw,
|
|
max_step_raw,
|
|
min_change_raw,
|
|
) = sys.argv[1:8]
|
|
|
|
|
|
def env_line(key, value):
|
|
return f"{key}={shlex.quote(str(value))}\n"
|
|
|
|
|
|
def as_int(value, default=0):
|
|
try:
|
|
return int(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def as_float(value, default=0.0):
|
|
try:
|
|
result = float(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
return result if math.isfinite(result) else default
|
|
|
|
|
|
result = json.loads(pathlib.Path(result_path).read_text())
|
|
calibration = result.get("output_delay_calibration") or {}
|
|
sync = result.get("sync") or {}
|
|
min_pairs = max(1, as_int(min_pairs_raw, 13))
|
|
max_abs_skew_ms = max(1.0, as_float(max_abs_skew_raw, 1000.0))
|
|
max_drift_ms = max(0.0, as_float(max_drift_raw, 80.0))
|
|
max_step_us = max(1, as_int(max_step_raw, 500_000))
|
|
min_change_us = max(0, as_int(min_change_raw, 5_000))
|
|
current_audio = as_int(result.get("audio_delay_us"), 0)
|
|
current_video = as_int(result.get("video_delay_us"), 0)
|
|
target_audio = as_int(calibration.get("audio_target_offset_us"), current_audio)
|
|
target_video = as_int(calibration.get("video_target_offset_us"), current_video)
|
|
paired = as_int(calibration.get("paired_event_count"), as_int(sync.get("paired_event_count"), 0))
|
|
drift_ms = as_float(calibration.get("drift_ms"), as_float(sync.get("drift_ms"), 0.0))
|
|
max_abs_skew_ms_observed = as_float(
|
|
calibration.get("max_abs_skew_ms"),
|
|
as_float(sync.get("p95_abs_skew_ms"), 0.0),
|
|
)
|
|
delta_audio = target_audio - current_audio
|
|
delta_video = target_video - current_video
|
|
raw_delta_us = as_int(
|
|
calibration.get("raw_device_delta_us"),
|
|
delta_video if abs(delta_video) >= abs(delta_audio) else -delta_audio,
|
|
)
|
|
reasons = []
|
|
|
|
if result.get("run_status") != 0:
|
|
reasons.append(f"seed probe exited {result.get('run_status')}")
|
|
if paired < min_pairs:
|
|
reasons.append(f"paired_event_count {paired} < {min_pairs}")
|
|
if max_abs_skew_ms_observed > max_abs_skew_ms:
|
|
reasons.append(
|
|
f"max_abs_skew_ms {max_abs_skew_ms_observed:.1f} > {max_abs_skew_ms:.1f}"
|
|
)
|
|
if abs(drift_ms) > max_drift_ms:
|
|
reasons.append(f"abs(drift_ms) {abs(drift_ms):.1f} > {max_drift_ms:.1f}")
|
|
if abs(raw_delta_us) > max_step_us:
|
|
reasons.append(f"raw delay correction {raw_delta_us:+d}us exceeds {max_step_us}us")
|
|
if target_audio < 0 or target_video < 0:
|
|
reasons.append(
|
|
f"direct confirmation delays must be non-negative, got audio={target_audio} video={target_video}"
|
|
)
|
|
if abs(delta_audio) < min_change_us and abs(delta_video) < min_change_us:
|
|
reasons.append(
|
|
f"target change audio={delta_audio:+d}us video={delta_video:+d}us is below {min_change_us}us"
|
|
)
|
|
|
|
ready = not reasons
|
|
values = {
|
|
"tune_ready": str(ready).lower(),
|
|
"tune_reason": "ready" if ready else "; ".join(reasons),
|
|
"tune_audio_delay_us": target_audio,
|
|
"tune_video_delay_us": target_video,
|
|
"tune_audio_delta_us": delta_audio,
|
|
"tune_video_delta_us": delta_video,
|
|
"tune_paired_event_count": paired,
|
|
"tune_drift_ms": f"{drift_ms:.3f}",
|
|
"tune_max_abs_skew_ms": f"{max_abs_skew_ms_observed:.3f}",
|
|
"tune_raw_delta_us": raw_delta_us,
|
|
}
|
|
with pathlib.Path(output_env_path).open("w") as handle:
|
|
for key, value in values.items():
|
|
handle.write(env_line(key, value))
|
|
PY
|
|
}
|
|
|
|
signal_readiness_passed() {
|
|
local artifact_dir=$1
|
|
local min_pairs=$2
|
|
python3 - <<'PY' "${artifact_dir}" "${min_pairs}"
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
artifact_dir = pathlib.Path(sys.argv[1])
|
|
min_pairs = int(sys.argv[2])
|
|
|
|
def load_json(path):
|
|
try:
|
|
return json.loads(path.read_text())
|
|
except Exception:
|
|
return {}
|
|
|
|
report = load_json(artifact_dir / "report.json")
|
|
paired = int(report.get("paired_event_count") or 0)
|
|
video = int(report.get("video_event_count") or 0)
|
|
audio = int(report.get("audio_event_count") or 0)
|
|
verdict = report.get("verdict") or {}
|
|
passed = paired >= min_pairs and video >= min_pairs and audio >= min_pairs and verdict.get("passed") is True
|
|
if not passed:
|
|
reason = (
|
|
f"signal readiness needs at least {min_pairs} paired coded events with sync pass; "
|
|
f"saw paired={paired}, video={video}, audio={audio}, sync={verdict.get('status', 'unknown')}"
|
|
)
|
|
print(reason)
|
|
raise SystemExit(0 if passed else 1)
|
|
PY
|
|
}
|
|
|
|
write_signal_readiness_attempt_result() {
|
|
local attempt=$1
|
|
local artifact_dir=$2
|
|
local run_status=$3
|
|
local run_log=$4
|
|
local min_pairs=$5
|
|
local output_json=$6
|
|
python3 - <<'PY' \
|
|
"${attempt}" \
|
|
"${artifact_dir}" \
|
|
"${run_status}" \
|
|
"${run_log}" \
|
|
"${min_pairs}" \
|
|
"${output_json}"
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
|
|
(
|
|
attempt_raw,
|
|
artifact_dir_raw,
|
|
run_status_raw,
|
|
run_log_raw,
|
|
min_pairs_raw,
|
|
output_json_raw,
|
|
) = sys.argv[1:]
|
|
|
|
def as_int(value, default=0):
|
|
try:
|
|
return int(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
|
|
def as_float(value, default=0.0):
|
|
try:
|
|
return float(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
|
|
def load_json(path):
|
|
try:
|
|
return json.loads(path.read_text())
|
|
except Exception:
|
|
return {}
|
|
|
|
def read_text(path):
|
|
try:
|
|
return path.read_text(errors="replace")
|
|
except Exception:
|
|
return ""
|
|
|
|
attempt = as_int(attempt_raw)
|
|
run_status = as_int(run_status_raw)
|
|
min_pairs = as_int(min_pairs_raw, 1)
|
|
run_log = pathlib.Path(run_log_raw)
|
|
artifact_dir = pathlib.Path(artifact_dir_raw) if artifact_dir_raw else None
|
|
if artifact_dir is None or not artifact_dir.exists():
|
|
match = re.findall(r"^artifact_dir:\s*(.+)$", read_text(run_log), flags=re.MULTILINE)
|
|
if match:
|
|
artifact_dir = pathlib.Path(match[-1].strip())
|
|
if artifact_dir is None:
|
|
artifact_dir = pathlib.Path("__missing_signal_readiness_artifact__")
|
|
|
|
report = load_json(artifact_dir / "report.json")
|
|
timeline = load_json(artifact_dir / "server-output-timeline.json")
|
|
error_log = artifact_dir / "analysis-error.log"
|
|
error_text = read_text(error_log)
|
|
if not error_text:
|
|
error_log = artifact_dir / "analysis-error.log.tmp"
|
|
error_text = read_text(error_log)
|
|
if not error_text:
|
|
error_text = read_text(run_log)
|
|
|
|
events = timeline.get("events") or []
|
|
server_event_count = len(events)
|
|
server_video_handoff_count = sum(
|
|
1
|
|
for event in events
|
|
if event.get("video_feed_unix_ns") is not None
|
|
or event.get("video_feed_monotonic_us") is not None
|
|
)
|
|
server_audio_handoff_count = sum(
|
|
1
|
|
for event in events
|
|
if event.get("audio_push_unix_ns") is not None
|
|
or event.get("audio_push_monotonic_us") is not None
|
|
)
|
|
|
|
verdict = report.get("verdict") or {}
|
|
signature_coverage = report.get("signature_coverage") or {}
|
|
video_events = as_int(report.get("video_event_count"))
|
|
audio_events = as_int(report.get("audio_event_count"))
|
|
paired_events = as_int(report.get("paired_event_count"))
|
|
coded_pairs = as_int(signature_coverage.get("paired_event_count"), paired_events)
|
|
expected_pairs = as_int(signature_coverage.get("expected_event_count"))
|
|
raw_activity_delta_ms = as_float(report.get("activity_start_delta_ms"))
|
|
analyzer_failure = ""
|
|
|
|
failure_match = re.search(
|
|
r"coded pulse common window removed one stream entirely;[^\n]*"
|
|
r"\(video=(?P<video>\d+)\s+audio=(?P<audio>\d+)\s+raw activity delta\s+"
|
|
r"(?P<delta>[+-]?\d+(?:\.\d+)?)\s+ms\)",
|
|
error_text,
|
|
)
|
|
if failure_match:
|
|
video_events = max(video_events, as_int(failure_match.group("video")))
|
|
audio_events = max(audio_events, as_int(failure_match.group("audio")))
|
|
raw_activity_delta_ms = as_float(failure_match.group("delta"))
|
|
analyzer_failure = failure_match.group(0)
|
|
|
|
pairs_match = re.search(
|
|
r"need at least\s+(?P<needed>\d+)\s+matching coded pulse pairs;\s+"
|
|
r"saw\s+(?P<paired>\d+);\s+raw activity delta was\s+"
|
|
r"(?P<delta>[+-]?\d+(?:\.\d+)?)\s+ms\s+"
|
|
r"\(video=(?P<video_time>\d+(?:\.\d+)?)s\s+audio=(?P<audio_time>\d+(?:\.\d+)?)s\)",
|
|
error_text,
|
|
)
|
|
if pairs_match:
|
|
paired_from_error = as_int(pairs_match.group("paired"))
|
|
paired_events = max(paired_events, paired_from_error)
|
|
coded_pairs = max(coded_pairs, paired_from_error)
|
|
video_events = max(video_events, paired_from_error)
|
|
audio_events = max(audio_events, paired_from_error)
|
|
raw_activity_delta_ms = as_float(pairs_match.group("delta"))
|
|
analyzer_failure = pairs_match.group(0)
|
|
|
|
sync_passed = verdict.get("passed") is True
|
|
paired_ready = paired_events >= min_pairs or coded_pairs >= min_pairs
|
|
video_ready = video_events >= min_pairs
|
|
audio_ready = audio_events >= min_pairs
|
|
passed = run_status == 0 and paired_ready and video_ready and audio_ready and sync_passed
|
|
|
|
reasons = []
|
|
if run_status != 0:
|
|
reasons.append(f"probe command exited {run_status}")
|
|
if server_event_count == 0:
|
|
reasons.append("server did not report generated coded events")
|
|
if server_video_handoff_count == 0:
|
|
reasons.append("server did not report video sink handoff events")
|
|
if server_audio_handoff_count == 0:
|
|
reasons.append("server did not report audio sink handoff events")
|
|
if video_events < min_pairs:
|
|
reasons.append(f"Tethys video coded events {video_events} < {min_pairs}")
|
|
if audio_events < min_pairs:
|
|
reasons.append(f"Tethys audio coded events {audio_events} < {min_pairs}")
|
|
if max(paired_events, coded_pairs) < min_pairs:
|
|
reasons.append(f"paired coded events {max(paired_events, coded_pairs)} < {min_pairs}")
|
|
if report and not sync_passed:
|
|
reasons.append(f"sync verdict did not pass: {verdict.get('status', 'unknown')}")
|
|
if analyzer_failure:
|
|
reasons.append(analyzer_failure)
|
|
if not reasons and passed:
|
|
reasons.append("ready")
|
|
elif not reasons:
|
|
reasons.append("signal readiness failed")
|
|
|
|
result = {
|
|
"schema": "lesavka.server-rc-signal-readiness-attempt.v1",
|
|
"attempt": attempt,
|
|
"passed": passed,
|
|
"reason": "; ".join(reasons),
|
|
"run_status": run_status,
|
|
"run_log": str(run_log),
|
|
"artifact_dir": str(artifact_dir) if artifact_dir.exists() else "",
|
|
"analysis_error_log": str(error_log) if error_log.exists() else "",
|
|
"counts": {
|
|
"server_events": server_event_count,
|
|
"server_video_handoffs": server_video_handoff_count,
|
|
"server_audio_handoffs": server_audio_handoff_count,
|
|
"video_events": video_events,
|
|
"audio_events": audio_events,
|
|
"paired_events": paired_events,
|
|
"coded_pairs": coded_pairs,
|
|
"expected_pairs": expected_pairs,
|
|
},
|
|
"layers": {
|
|
"server_generated_video": server_event_count > 0,
|
|
"server_generated_audio": server_event_count > 0,
|
|
"server_video_handoff": server_video_handoff_count > 0,
|
|
"server_audio_handoff": server_audio_handoff_count > 0,
|
|
"tethys_video_detected": video_events > 0,
|
|
"tethys_audio_detected": audio_events > 0,
|
|
"tethys_video_ready": video_ready,
|
|
"tethys_audio_ready": audio_ready,
|
|
"paired_coded_ready": paired_ready,
|
|
"analyzer_sync_passed": sync_passed,
|
|
},
|
|
"sync": {
|
|
"status": verdict.get("status", "unknown"),
|
|
"passed": sync_passed,
|
|
"p95_abs_skew_ms": as_float(verdict.get("p95_abs_skew_ms")),
|
|
"median_skew_ms": as_float(report.get("median_skew_ms")),
|
|
"drift_ms": as_float(report.get("drift_ms")),
|
|
"raw_activity_delta_ms": raw_activity_delta_ms,
|
|
},
|
|
"analyzer_failure": analyzer_failure,
|
|
}
|
|
pathlib.Path(output_json_raw).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
|
print(result["reason"])
|
|
raise SystemExit(0 if passed else 1)
|
|
PY
|
|
}
|
|
|
|
write_signal_readiness_attempts_summary() {
|
|
local output_json=$1
|
|
shift || true
|
|
python3 - <<'PY' "${output_json}" "$@"
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
output = pathlib.Path(sys.argv[1])
|
|
attempts = []
|
|
for raw in sys.argv[2:]:
|
|
path = pathlib.Path(raw)
|
|
try:
|
|
attempts.append(json.loads(path.read_text()))
|
|
except Exception:
|
|
attempts.append({
|
|
"schema": "lesavka.server-rc-signal-readiness-attempt.v1",
|
|
"attempt_json": str(path),
|
|
"passed": False,
|
|
"reason": "attempt result could not be read",
|
|
})
|
|
passed_attempt = next((attempt for attempt in attempts if attempt.get("passed")), None)
|
|
summary = {
|
|
"schema": "lesavka.server-rc-signal-readiness-summary.v1",
|
|
"passed": passed_attempt is not None,
|
|
"passed_attempt": passed_attempt.get("attempt") if passed_attempt else None,
|
|
"attempt_count": len(attempts),
|
|
"final_reason": (
|
|
"ready"
|
|
if passed_attempt
|
|
else (attempts[-1].get("reason") if attempts else "no readiness attempts ran")
|
|
),
|
|
"attempts": attempts,
|
|
}
|
|
output.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
|
|
PY
|
|
}
|
|
|
|
write_signal_readiness_failure() {
|
|
local mode=$1
|
|
local width=$2
|
|
local height=$3
|
|
local fps=$4
|
|
local video_delay_us=$5
|
|
local audio_delay_us=$6
|
|
local readiness_status=$7
|
|
local readiness_log=$8
|
|
local readiness_artifact_dir=$9
|
|
local readiness_attempts_json=${10}
|
|
local readiness_reason=${11}
|
|
local output_json=${12}
|
|
python3 - <<'PY' \
|
|
"${mode}" \
|
|
"${width}" \
|
|
"${height}" \
|
|
"${fps}" \
|
|
"${video_delay_us}" \
|
|
"${audio_delay_us}" \
|
|
"${readiness_status}" \
|
|
"${readiness_log}" \
|
|
"${readiness_artifact_dir}" \
|
|
"${readiness_attempts_json}" \
|
|
"${readiness_reason}" \
|
|
"${output_json}"
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
(
|
|
mode,
|
|
width_raw,
|
|
height_raw,
|
|
fps_raw,
|
|
video_delay_raw,
|
|
audio_delay_raw,
|
|
readiness_status_raw,
|
|
readiness_log,
|
|
readiness_artifact_dir,
|
|
readiness_attempts_json,
|
|
readiness_reason,
|
|
output_json,
|
|
) = sys.argv[1:]
|
|
|
|
def as_int(value, default=0):
|
|
try:
|
|
return int(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
|
|
def load_json(path):
|
|
try:
|
|
return json.loads(pathlib.Path(path).read_text())
|
|
except Exception:
|
|
return {}
|
|
|
|
report = load_json(pathlib.Path(readiness_artifact_dir) / "report.json")
|
|
verdict = report.get("verdict") or {}
|
|
signature_coverage = report.get("signature_coverage") or {}
|
|
readiness_attempts = load_json(readiness_attempts_json).get("attempts") if readiness_attempts_json else None
|
|
result = {
|
|
"schema": "lesavka.server-rc-mode-result.v1",
|
|
"mode": mode,
|
|
"width": as_int(width_raw),
|
|
"height": as_int(height_raw),
|
|
"fps": as_int(fps_raw),
|
|
"audio_delay_us": as_int(audio_delay_raw),
|
|
"video_delay_us": as_int(video_delay_raw),
|
|
"run_status": as_int(readiness_status_raw),
|
|
"run_log": readiness_log,
|
|
"artifact_dir": readiness_artifact_dir,
|
|
"report_json": str(pathlib.Path(readiness_artifact_dir) / "report.json"),
|
|
"passed": False,
|
|
"failure_reasons": [f"signal readiness did not pass: {readiness_reason}"],
|
|
"signal_readiness": {
|
|
"passed": False,
|
|
"reason": readiness_reason,
|
|
"artifact_dir": readiness_artifact_dir,
|
|
"run_log": readiness_log,
|
|
"attempts_json": readiness_attempts_json,
|
|
"attempts": readiness_attempts or [],
|
|
},
|
|
"sync": {
|
|
"passed": verdict.get("passed") is True,
|
|
"status": verdict.get("status", "unknown"),
|
|
"reason": verdict.get("reason", ""),
|
|
"p95_abs_skew_ms": float(verdict.get("p95_abs_skew_ms") or 0.0),
|
|
"median_skew_ms": float(report.get("median_skew_ms") or 0.0),
|
|
"drift_ms": float(report.get("drift_ms") or 0.0),
|
|
"activity_start_delta_ms": float(report.get("activity_start_delta_ms") or 0.0),
|
|
"first_skew_ms": float(report.get("first_skew_ms") or 0.0),
|
|
"video_event_count": as_int(report.get("video_event_count")),
|
|
"audio_event_count": as_int(report.get("audio_event_count")),
|
|
"paired_event_count": as_int(report.get("paired_event_count")),
|
|
"signature_expected_event_count": as_int(signature_coverage.get("expected_event_count")),
|
|
"signature_paired_event_count": as_int(signature_coverage.get("paired_event_count")),
|
|
"signature_missing_event_ids": signature_coverage.get("missing_event_ids") or [],
|
|
"signature_missing_codes": signature_coverage.get("missing_codes") or [],
|
|
"signature_unknown_pair_identity_count": as_int(signature_coverage.get("unknown_pair_identity_count")),
|
|
},
|
|
"freshness": {
|
|
"status": "unknown",
|
|
"reason": "measured probe skipped because signal readiness failed",
|
|
"worst_event_age_with_uncertainty_ms": 0.0,
|
|
},
|
|
"smoothness": {},
|
|
"output_delay_calibration": {},
|
|
}
|
|
pathlib.Path(output_json).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
|
PY
|
|
}
|
|
|
|
discover_local_webcam_modes() {
|
|
if ! command -v python3 >/dev/null 2>&1; then
|
|
printf 'LESAVKA_SERVER_RC_MODES=auto requires python3 for local webcam mode discovery.\n' >&2
|
|
exit 64
|
|
fi
|
|
if ! command -v v4l2-ctl >/dev/null 2>&1; then
|
|
printf 'LESAVKA_SERVER_RC_MODES=auto requires v4l2-ctl for local webcam mode discovery.\n' >&2
|
|
exit 64
|
|
fi
|
|
python3 - <<'PY' \
|
|
"${LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES}" \
|
|
"${LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS}" \
|
|
"${LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX}" \
|
|
"${LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX}"
|
|
import glob
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
sizes_raw, fps_raw, include_raw, exclude_raw = sys.argv[1:5]
|
|
|
|
def parse_sizes(raw):
|
|
sizes = set()
|
|
for entry in raw.split(","):
|
|
entry = entry.strip()
|
|
if not entry:
|
|
continue
|
|
match = re.fullmatch(r"(\d+)x(\d+)", entry)
|
|
if not match:
|
|
raise SystemExit(f"invalid discovery size {entry!r}; expected WIDTHxHEIGHT")
|
|
sizes.add((int(match.group(1)), int(match.group(2))))
|
|
return sizes
|
|
|
|
def parse_fps(raw):
|
|
values = set()
|
|
for entry in raw.split(","):
|
|
entry = entry.strip()
|
|
if not entry:
|
|
continue
|
|
if not entry.isdigit():
|
|
raise SystemExit(f"invalid discovery fps {entry!r}; expected integer FPS")
|
|
values.add(int(entry))
|
|
return values
|
|
|
|
allowed_sizes = parse_sizes(sizes_raw)
|
|
allowed_fps = parse_fps(fps_raw)
|
|
include = re.compile(include_raw, re.IGNORECASE) if include_raw else None
|
|
exclude = re.compile(exclude_raw, re.IGNORECASE) if exclude_raw else None
|
|
all_modes = set()
|
|
camera_modes = []
|
|
|
|
for dev in sorted(glob.glob("/dev/video*")):
|
|
try:
|
|
info = subprocess.run(
|
|
["v4l2-ctl", "-d", dev, "--info"],
|
|
check=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
text=True,
|
|
).stdout
|
|
formats = subprocess.run(
|
|
["v4l2-ctl", "-d", dev, "--list-formats-ext"],
|
|
check=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
text=True,
|
|
).stdout
|
|
except Exception:
|
|
continue
|
|
|
|
card = dev
|
|
for line in info.splitlines():
|
|
if "Card type" in line:
|
|
card = line.split(":", 1)[1].strip()
|
|
break
|
|
label = f"{card} {dev}"
|
|
if exclude and exclude.search(label):
|
|
continue
|
|
if include and not include.search(label):
|
|
continue
|
|
|
|
current_size = None
|
|
modes = set()
|
|
for line in formats.splitlines():
|
|
size_match = re.search(r"Size:\s+Discrete\s+(\d+)x(\d+)", line)
|
|
if size_match:
|
|
current_size = (int(size_match.group(1)), int(size_match.group(2)))
|
|
continue
|
|
fps_match = re.search(r"Interval:\s+Discrete\s+[^()]+\((\d+(?:\.\d+)?) fps\)", line)
|
|
if not fps_match or current_size not in allowed_sizes:
|
|
continue
|
|
fps_float = float(fps_match.group(1))
|
|
fps = int(round(fps_float))
|
|
if abs(fps_float - fps) > 0.05 or fps not in allowed_fps:
|
|
continue
|
|
modes.add((current_size[0], current_size[1], fps))
|
|
|
|
if modes:
|
|
all_modes.update(modes)
|
|
camera_modes.append((label, modes))
|
|
|
|
if not all_modes:
|
|
print(
|
|
"no matching local webcam modes found; "
|
|
f"sizes={sizes_raw} fps={fps_raw} include={include_raw!r} exclude={exclude_raw!r}",
|
|
file=sys.stderr,
|
|
)
|
|
raise SystemExit(64)
|
|
|
|
for label, modes in camera_modes:
|
|
rendered = ",".join(f"{w}x{h}@{fps}" for w, h, fps in sorted(modes))
|
|
print(f" -> local webcam {label}: {rendered}", file=sys.stderr)
|
|
|
|
print(",".join(f"{w}x{h}@{fps}" for w, h, fps in sorted(all_modes)), end="")
|
|
PY
|
|
}
|
|
|
|
clear_remote_sudo_password() {
|
|
LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD=""
|
|
}
|
|
trap clear_remote_sudo_password EXIT
|
|
|
|
ensure_remote_sudo_password() {
|
|
[[ "${LESAVKA_SERVER_RC_RECONFIGURE}" != "0" ]] || return 0
|
|
[[ -z "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}" ]] || return 0
|
|
[[ -n "${LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD}" ]] && return 0
|
|
|
|
if [[ "${LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY}" == "0" ]]; then
|
|
printf 'remote sudo password is required for automatic reconfigure; set LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD or leave LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY=1.\n' >&2
|
|
exit 64
|
|
fi
|
|
if [[ ! -t 0 ]]; then
|
|
printf 'remote sudo password is required, but stdin is not a terminal.\n' >&2
|
|
printf 'Re-run from a terminal or set LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD in the environment.\n' >&2
|
|
exit 64
|
|
fi
|
|
|
|
printf 'Theia sudo password for %s: ' "${LESAVKA_SERVER_HOST}" >&2
|
|
IFS= read -r -s LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD
|
|
printf '\n' >&2
|
|
}
|
|
|
|
run_remote_root_script() {
|
|
local description=$1
|
|
shift
|
|
local script_file remote_wrapper ssh_cmd status
|
|
script_file="$(mktemp)"
|
|
cat >"${script_file}"
|
|
|
|
remote_wrapper='set -euo pipefail
|
|
read -r __lesavka_sudo_password
|
|
__lesavka_script=$(mktemp)
|
|
cleanup() { rm -f "$__lesavka_script"; }
|
|
trap cleanup EXIT
|
|
cat >"$__lesavka_script"
|
|
printf "%s\n" "$__lesavka_sudo_password" | sudo -S -p "" -v
|
|
printf "%s\n" "$__lesavka_sudo_password" | sudo -S -p "" bash "$__lesavka_script" "$@"
|
|
'
|
|
printf -v ssh_cmd 'bash -c %q _' "${remote_wrapper}"
|
|
for arg in "$@"; do
|
|
printf -v ssh_cmd '%s %q' "${ssh_cmd}" "${arg}"
|
|
done
|
|
|
|
set +e
|
|
{
|
|
printf '%s\n' "${LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD}"
|
|
cat "${script_file}"
|
|
} | ssh ${SSH_OPTS} "${LESAVKA_SERVER_HOST}" "${ssh_cmd}"
|
|
status=$?
|
|
set -e
|
|
rm -f "${script_file}"
|
|
if [[ "${status}" -ne 0 ]]; then
|
|
printf '%s failed on %s with exit %s\n' "${description}" "${LESAVKA_SERVER_HOST}" "${status}" >&2
|
|
fi
|
|
return "${status}"
|
|
}
|
|
|
|
prime_remote_sudo() {
|
|
[[ "${LESAVKA_SERVER_RC_RECONFIGURE}" != "0" ]] || return 0
|
|
[[ -z "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}" ]] || return 0
|
|
ensure_remote_sudo_password
|
|
echo "==> priming remote sudo on ${LESAVKA_SERVER_HOST}"
|
|
run_remote_root_script "remote sudo prime" <<'REMOTE_SUDO_PRIME'
|
|
set -euo pipefail
|
|
true
|
|
REMOTE_SUDO_PRIME
|
|
}
|
|
|
|
sleep_start_delay() {
|
|
[[ "${LESAVKA_SERVER_RC_START_DELAY_SECONDS}" != "0" ]] || return 0
|
|
if [[ ! "${LESAVKA_SERVER_RC_START_DELAY_SECONDS}" =~ ^[0-9]+([.][0-9]+)?$ ]]; then
|
|
printf 'LESAVKA_SERVER_RC_START_DELAY_SECONDS must be a non-negative number, got %s\n' "${LESAVKA_SERVER_RC_START_DELAY_SECONDS}" >&2
|
|
exit 64
|
|
fi
|
|
echo "==> delaying server-to-RC matrix start for ${LESAVKA_SERVER_RC_START_DELAY_SECONDS}s"
|
|
echo " ↪ remote sudo has already been primed; sleeping before prebuild/reconfigure/capture"
|
|
sleep "${LESAVKA_SERVER_RC_START_DELAY_SECONDS}"
|
|
}
|
|
|
|
prebuild_probe_tools() {
|
|
[[ "${LESAVKA_SERVER_RC_PROBE_PREBUILD}" != "0" ]] || return 0
|
|
echo "==> prebuilding relay control/analyzer once for the mode matrix"
|
|
(
|
|
cd "${REPO_ROOT}"
|
|
cargo build -p lesavka_client --bin lesavka-sync-analyze --bin lesavka-relayctl
|
|
)
|
|
}
|
|
|
|
reconfigure_server_mode() {
|
|
local mode=$1
|
|
local width=$2
|
|
local height=$3
|
|
local fps=$4
|
|
local audio_delay_us=${5:-$(lookup_audio_delay_us "${mode}")}
|
|
local video_delay_us=${6:-$(lookup_video_delay_us "${mode}")}
|
|
[[ "${LESAVKA_SERVER_RC_RECONFIGURE}" != "0" ]] || return 0
|
|
|
|
echo "==> reconfiguring ${LESAVKA_SERVER_HOST} UVC gadget for ${mode}"
|
|
if [[ -n "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}" ]]; then
|
|
LESAVKA_MODE="${mode}" \
|
|
LESAVKA_UVC_WIDTH="${width}" \
|
|
LESAVKA_UVC_HEIGHT="${height}" \
|
|
LESAVKA_UVC_FPS="${fps}" \
|
|
LESAVKA_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US="${audio_delay_us}" \
|
|
LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US="${video_delay_us}" \
|
|
bash -c "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY}" != "runtime" ]]; then
|
|
printf 'unsupported LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY=%s\n' "${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY}" >&2
|
|
printf 'Use runtime for fast mode changes, or LESAVKA_SERVER_RC_RECONFIGURE_COMMAND for a custom install/reconfigure command.\n' >&2
|
|
return 64
|
|
fi
|
|
|
|
local interval
|
|
interval=$((10000000 / fps))
|
|
run_remote_root_script "runtime UVC reconfigure for ${mode}" \
|
|
"${mode}" \
|
|
"${width}" \
|
|
"${height}" \
|
|
"${fps}" \
|
|
"${interval}" \
|
|
"${LESAVKA_SERVER_RC_RECONFIGURE_CODEC}" \
|
|
"${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET}" \
|
|
"${LESAVKA_SERVER_RC_FORCE_GADGET_REBUILD}" \
|
|
"${LESAVKA_SERVER_RC_RECONFIGURE_SETTLE_SECONDS}" \
|
|
"${LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE}" \
|
|
"${audio_delay_us}" \
|
|
"${video_delay_us}" \
|
|
"${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}" \
|
|
"${LESAVKA_SERVER_RC_MODE_DELAYS_US}" <<'REMOTE_RECONFIGURE'
|
|
set -euo pipefail
|
|
mode=$1
|
|
width=$2
|
|
height=$3
|
|
fps=$4
|
|
interval=$5
|
|
codec=$6
|
|
allow_gadget_reset=$7
|
|
force_gadget_rebuild=$8
|
|
settle_seconds=$9
|
|
verbose=${10}
|
|
audio_delay_us=${11}
|
|
video_delay_us=${12}
|
|
audio_delay_map=${13}
|
|
video_delay_map=${14}
|
|
|
|
set_env_value() {
|
|
local file=$1
|
|
local key=$2
|
|
local value=$3
|
|
local tmp
|
|
tmp=$(mktemp)
|
|
if [[ -f "${file}" ]]; then
|
|
awk -v key="${key}" -v value="${value}" '
|
|
BEGIN { wrote = 0 }
|
|
$0 ~ "^" key "=" {
|
|
print key "=" value
|
|
wrote = 1
|
|
next
|
|
}
|
|
{ print }
|
|
END {
|
|
if (!wrote) {
|
|
print key "=" value
|
|
}
|
|
}
|
|
' "${file}" >"${tmp}"
|
|
else
|
|
printf '%s=%s\n' "${key}" "${value}" >"${tmp}"
|
|
fi
|
|
install -m 0644 "${tmp}" "${file}"
|
|
rm -f "${tmp}"
|
|
}
|
|
|
|
if [[ ! -x /usr/local/bin/lesavka-core.sh ]]; then
|
|
printf 'missing /usr/local/bin/lesavka-core.sh; run the server installer once before using fast runtime reconfigure.\n' >&2
|
|
exit 65
|
|
fi
|
|
if [[ ! -x /usr/local/bin/lesavka-server || ! -x /usr/local/bin/lesavka-uvc ]]; then
|
|
printf 'missing installed Lesavka binaries; run the server installer once before using fast runtime reconfigure.\n' >&2
|
|
exit 65
|
|
fi
|
|
|
|
install -d -m 0755 /etc/lesavka
|
|
touch /etc/lesavka/server.env /etc/lesavka/uvc.env
|
|
|
|
set_env_value /etc/lesavka/server.env LESAVKA_CAM_OUTPUT uvc
|
|
set_env_value /etc/lesavka/server.env LESAVKA_UVC_CODEC "${codec}"
|
|
set_env_value /etc/lesavka/server.env LESAVKA_UVC_WIDTH "${width}"
|
|
set_env_value /etc/lesavka/server.env LESAVKA_UVC_HEIGHT "${height}"
|
|
set_env_value /etc/lesavka/server.env LESAVKA_UVC_FPS "${fps}"
|
|
set_env_value /etc/lesavka/server.env LESAVKA_UVC_INTERVAL "${interval}"
|
|
set_env_value /etc/lesavka/server.env LESAVKA_UPSTREAM_AUDIO_PLAYOUT_MODE_OFFSETS_US "${audio_delay_map}"
|
|
set_env_value /etc/lesavka/server.env LESAVKA_UPSTREAM_VIDEO_PLAYOUT_MODE_OFFSETS_US "${video_delay_map}"
|
|
set_env_value /etc/lesavka/server.env LESAVKA_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US "${audio_delay_us}"
|
|
set_env_value /etc/lesavka/server.env LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US "${video_delay_us}"
|
|
|
|
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_CODEC "${codec}"
|
|
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_WIDTH "${width}"
|
|
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_HEIGHT "${height}"
|
|
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_FPS "${fps}"
|
|
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_INTERVAL "${interval}"
|
|
|
|
printf ' ↪ fast runtime env updated: CAM_OUTPUT=uvc UVC_MODE=%s codec=%s\n' "${mode}" "${codec}"
|
|
systemctl daemon-reload
|
|
systemctl stop lesavka-server >/dev/null 2>&1 || true
|
|
systemctl stop lesavka-uvc >/dev/null 2>&1 || true
|
|
systemctl reset-failed lesavka-core lesavka-uvc lesavka-server >/dev/null 2>&1 || true
|
|
|
|
if [[ "${allow_gadget_reset}" != "0" && "${force_gadget_rebuild}" != "0" ]]; then
|
|
printf ' ↪ cycling UVC gadget descriptors for %s\n' "${mode}"
|
|
mode_log_id=$(printf '%s' "${mode}" | tr -c '[:alnum:]_.-' '_')
|
|
core_log="/tmp/lesavka-core-reconfigure-${mode_log_id}.log"
|
|
core_cmd=(
|
|
env
|
|
LESAVKA_ALLOW_GADGET_RESET=1 \
|
|
LESAVKA_FORCE_GADGET_REBUILD=1 \
|
|
LESAVKA_ATTACH_WRITE_UDC=1 \
|
|
LESAVKA_DETACH_CLEAR_UDC=1 \
|
|
LESAVKA_UVC_FALLBACK=0 \
|
|
LESAVKA_UVC_CODEC="${codec}" \
|
|
LESAVKA_UVC_WIDTH="${width}" \
|
|
LESAVKA_UVC_HEIGHT="${height}" \
|
|
LESAVKA_UVC_FPS="${fps}" \
|
|
LESAVKA_UVC_INTERVAL="${interval}" \
|
|
/usr/local/bin/lesavka-core.sh
|
|
)
|
|
if [[ "${verbose}" != "0" ]]; then
|
|
"${core_cmd[@]}"
|
|
else
|
|
if "${core_cmd[@]}" >"${core_log}" 2>&1; then
|
|
printf ' ↪ lesavka-core reconfigure log: %s\n' "${core_log}"
|
|
else
|
|
rc=$?
|
|
printf 'lesavka-core reconfigure failed; tail of %s follows\n' "${core_log}" >&2
|
|
tail -n 80 "${core_log}" >&2 || true
|
|
exit "${rc}"
|
|
fi
|
|
fi
|
|
elif [[ "${allow_gadget_reset}" != "0" ]]; then
|
|
printf ' ↪ gadget reset allowed, but force rebuild disabled; refreshing services only\n'
|
|
else
|
|
printf ' ↪ preserving attached UVC gadget; descriptors may remain on the previous mode\n'
|
|
fi
|
|
|
|
systemctl start lesavka-uvc
|
|
systemctl restart lesavka-server
|
|
sleep "${settle_seconds}"
|
|
systemctl is-active lesavka-core lesavka-uvc lesavka-server >/dev/null
|
|
printf ' ↪ services active after %s reconfigure\n' "${mode}"
|
|
REMOTE_RECONFIGURE
|
|
}
|
|
|
|
wait_tethys_media_ready() {
|
|
local mode=$1
|
|
local width=$2
|
|
local height=$3
|
|
local fps=$4
|
|
[[ "${LESAVKA_SERVER_RC_WAIT_TETHYS_READY}" != "0" ]] || return 0
|
|
|
|
echo "==> waiting for Tethys media endpoints for ${mode}"
|
|
ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
|
|
"${mode}" \
|
|
"${width}" \
|
|
"${height}" \
|
|
"${fps}" \
|
|
"${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}" \
|
|
"${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}" \
|
|
"${REMOTE_CAPTURE_STACK}" \
|
|
"${REMOTE_AUDIO_SOURCE}" <<'REMOTE_TETHYS_READY'
|
|
set -euo pipefail
|
|
mode=$1
|
|
width=$2
|
|
height=$3
|
|
fps=$4
|
|
timeout_seconds=$5
|
|
settle_seconds=$6
|
|
capture_stack=$7
|
|
audio_source=$8
|
|
|
|
sleep "${settle_seconds}"
|
|
|
|
find_lesavka_video_device() {
|
|
if [[ -d /dev/v4l/by-id ]]; then
|
|
while IFS= read -r path; do
|
|
[[ -e "${path}" ]] || continue
|
|
printf '%s\n' "${path}"
|
|
return 0
|
|
done < <(find /dev/v4l/by-id -maxdepth 1 -type l -name '*Lesavka*video-index0' 2>/dev/null | sort)
|
|
fi
|
|
|
|
if command -v v4l2-ctl >/dev/null 2>&1; then
|
|
v4l2-ctl --list-devices 2>/dev/null \
|
|
| awk '
|
|
BEGIN { want=0 }
|
|
/Lesavka Composite|Lesavka.*UVC/ { want=1; next }
|
|
/^[^ \t]/ { want=0 }
|
|
want && /^[ \t]+\/dev\/video[0-9]+/ {
|
|
gsub(/^[ \t]+/, "", $0)
|
|
print
|
|
exit
|
|
}
|
|
'
|
|
fi
|
|
}
|
|
|
|
video_ready() {
|
|
local dev=$1
|
|
[[ -n "${dev}" ]] || return 1
|
|
[[ -e "${dev}" ]] || return 1
|
|
if ! command -v v4l2-ctl >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
local listing
|
|
listing="$(v4l2-ctl -d "${dev}" --list-formats-ext 2>/dev/null || true)"
|
|
grep -q "Size: Discrete ${width}x${height}" <<<"${listing}" || return 1
|
|
grep -Eq "(^|[^0-9])${fps}(\\.0+)? fps" <<<"${listing}" || return 1
|
|
}
|
|
|
|
pulse_ready() {
|
|
if ! command -v pactl >/dev/null 2>&1; then
|
|
return 1
|
|
fi
|
|
if [[ "${audio_source}" == pulse:* ]]; then
|
|
local requested=${audio_source#pulse:}
|
|
pactl list short sources 2>/dev/null | awk -v requested="${requested}" '$2 == requested { found=1 } END { exit found ? 0 : 1 }'
|
|
return
|
|
fi
|
|
pactl list short sources 2>/dev/null | grep -Eq 'alsa_input\..*Lesavka_Composite|Lesavka_Composite'
|
|
}
|
|
|
|
alsa_ready() {
|
|
if [[ "${audio_source}" == alsa:* ]]; then
|
|
[[ -e "${audio_source#alsa:}" ]] || return 1
|
|
return 0
|
|
fi
|
|
command -v arecord >/dev/null 2>&1 || return 1
|
|
arecord -l 2>/dev/null | grep -Eq 'Lesavka|UAC2_Gadget|UAC2Gadget|Composite'
|
|
}
|
|
|
|
audio_ready() {
|
|
case "${capture_stack}" in
|
|
pulse)
|
|
pulse_ready
|
|
;;
|
|
alsa)
|
|
alsa_ready
|
|
;;
|
|
auto)
|
|
pulse_ready || alsa_ready
|
|
;;
|
|
pwpipe)
|
|
command -v pw-dump >/dev/null 2>&1 && pw-dump 2>/dev/null | grep -Eq 'Lesavka_Composite|Lesavka Composite'
|
|
;;
|
|
*)
|
|
pulse_ready || alsa_ready
|
|
;;
|
|
esac
|
|
}
|
|
|
|
deadline=$((SECONDS + timeout_seconds))
|
|
last_video='none'
|
|
last_audio='none'
|
|
while (( SECONDS <= deadline )); do
|
|
video_dev="$(find_lesavka_video_device || true)"
|
|
last_video="${video_dev:-none}"
|
|
if [[ -n "${video_dev}" ]] && video_ready "${video_dev}"; then
|
|
if audio_ready; then
|
|
printf ' ↪ Tethys media ready: video=%s mode=%s audio_stack=%s\n' "${video_dev}" "${mode}" "${capture_stack}"
|
|
exit 0
|
|
fi
|
|
last_audio='not-ready'
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
|
|
printf 'timed out waiting for Tethys Lesavka media endpoints for %s after %ss\n' "${mode}" "${timeout_seconds}" >&2
|
|
printf 'last video candidate: %s\n' "${last_video}" >&2
|
|
printf 'last audio candidate: %s\n' "${last_audio}" >&2
|
|
if command -v v4l2-ctl >/dev/null 2>&1; then
|
|
v4l2-ctl --list-devices >&2 || true
|
|
if [[ "${last_video}" != "none" ]]; then
|
|
v4l2-ctl -d "${last_video}" --list-formats-ext >&2 || true
|
|
fi
|
|
fi
|
|
if command -v pactl >/dev/null 2>&1; then
|
|
pactl list short sources | grep -iE 'lesavka|uac|composite' >&2 || true
|
|
fi
|
|
if command -v arecord >/dev/null 2>&1; then
|
|
arecord -l >&2 || true
|
|
fi
|
|
exit 70
|
|
REMOTE_TETHYS_READY
|
|
}
|
|
|
|
write_mode_result() {
|
|
local mode=$1
|
|
local width=$2
|
|
local height=$3
|
|
local fps=$4
|
|
local video_delay_us=$5
|
|
local audio_delay_us=$6
|
|
local run_status=$7
|
|
local run_log=$8
|
|
local artifact_dir=$9
|
|
local output_json=${10}
|
|
|
|
python3 - <<'PY' \
|
|
"${mode}" \
|
|
"${width}" \
|
|
"${height}" \
|
|
"${fps}" \
|
|
"${video_delay_us}" \
|
|
"${audio_delay_us}" \
|
|
"${run_status}" \
|
|
"${run_log}" \
|
|
"${artifact_dir}" \
|
|
"${output_json}" \
|
|
"${LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS}" \
|
|
"${LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS}" \
|
|
"${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS}" \
|
|
"${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS}" \
|
|
"${LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS}" \
|
|
"${LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS}" \
|
|
"${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES}" \
|
|
"${LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES}" \
|
|
"${LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES}" \
|
|
"${LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS}" \
|
|
"${LESAVKA_SERVER_RC_MIN_CODED_PAIRS}" \
|
|
"${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS}" \
|
|
"${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS}"
|
|
import json
|
|
import math
|
|
import pathlib
|
|
import sys
|
|
|
|
(
|
|
mode,
|
|
width_raw,
|
|
height_raw,
|
|
fps_raw,
|
|
video_delay_raw,
|
|
audio_delay_raw,
|
|
run_status_raw,
|
|
run_log,
|
|
artifact_dir_raw,
|
|
output_json,
|
|
require_sync_raw,
|
|
require_freshness_raw,
|
|
max_video_hiccups_raw,
|
|
max_audio_hiccups_raw,
|
|
max_video_jitter_raw,
|
|
max_audio_jitter_raw,
|
|
max_missing_raw,
|
|
max_undecodable_raw,
|
|
max_duplicates_raw,
|
|
max_low_rms_raw,
|
|
min_coded_pairs_raw,
|
|
require_all_coded_pairs_raw,
|
|
require_smoothness_raw,
|
|
) = sys.argv[1:]
|
|
|
|
|
|
def as_bool(value):
|
|
return str(value).strip().lower() not in {"", "0", "false", "no", "off"}
|
|
|
|
|
|
def as_int(value, default=0):
|
|
try:
|
|
return int(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def as_float(value, default=0.0):
|
|
try:
|
|
result = float(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
return result if math.isfinite(result) else default
|
|
|
|
|
|
def load_json(path):
|
|
try:
|
|
return json.loads(pathlib.Path(path).read_text())
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def nested(mapping, *keys, default=None):
|
|
value = mapping
|
|
for key in keys:
|
|
if not isinstance(value, dict):
|
|
return default
|
|
value = value.get(key)
|
|
return default if value is None else value
|
|
|
|
|
|
artifact_dir = pathlib.Path(artifact_dir_raw) if artifact_dir_raw else pathlib.Path()
|
|
report = load_json(artifact_dir / "report.json")
|
|
correlation = load_json(artifact_dir / "output-delay-correlation.json")
|
|
calibration = load_json(artifact_dir / "output-delay-calibration.json")
|
|
timing_map = correlation.get("timing_map") or {}
|
|
freshness = correlation.get("freshness") or {}
|
|
capture_timebase = freshness.get("capture_timebase") or {}
|
|
smoothness = correlation.get("smoothness") or {}
|
|
event_visibility = correlation.get("event_visibility") or {}
|
|
video = smoothness.get("video") or {}
|
|
audio = smoothness.get("audio") or {}
|
|
audio_cadence = audio.get("packet_cadence") or {}
|
|
audio_rms = audio.get("rms_continuity") or {}
|
|
verdict = report.get("verdict") or {}
|
|
signature_coverage = report.get("signature_coverage") or {}
|
|
|
|
sync_pass = verdict.get("passed") is True
|
|
freshness_status = freshness.get("status", "unknown")
|
|
freshness_pass = freshness_status == "pass"
|
|
run_status = as_int(run_status_raw, 1)
|
|
|
|
video_hiccups = as_int(video.get("hiccup_count"), 0)
|
|
audio_hiccups = as_int(audio_cadence.get("hiccup_count"), 0)
|
|
video_jitter = as_float(nested(video, "jitter_stats", "p95_jitter_ms"), 0.0)
|
|
audio_jitter = as_float(nested(audio_cadence, "jitter_stats", "p95_jitter_ms"), 0.0)
|
|
missing = as_int(video.get("estimated_missing_frames"), 0)
|
|
undecodable = as_int(video.get("undecodable_frames"), 0)
|
|
duplicates = as_int(video.get("duplicate_frames"), 0)
|
|
low_rms = as_int(audio_rms.get("low_rms_window_count"), 0)
|
|
signature_expected = as_int(signature_coverage.get("expected_event_count"), 0)
|
|
signature_paired = as_int(signature_coverage.get("paired_event_count"), 0)
|
|
signature_unknown = as_int(signature_coverage.get("unknown_pair_identity_count"), 0)
|
|
min_coded_pairs = as_int(min_coded_pairs_raw, 0)
|
|
require_all_coded_pairs = as_bool(require_all_coded_pairs_raw)
|
|
require_smoothness = as_bool(require_smoothness_raw)
|
|
pair_confidences = []
|
|
for event in report.get("paired_events") or []:
|
|
if not isinstance(event, dict):
|
|
continue
|
|
try:
|
|
confidence = float(event.get("confidence"))
|
|
except Exception:
|
|
continue
|
|
if math.isfinite(confidence):
|
|
pair_confidences.append(confidence)
|
|
|
|
|
|
def median(values, default=0.0):
|
|
if not values:
|
|
return default
|
|
ordered = sorted(values)
|
|
mid = len(ordered) // 2
|
|
if len(ordered) % 2:
|
|
return ordered[mid]
|
|
return (ordered[mid - 1] + ordered[mid]) / 2.0
|
|
|
|
|
|
activity_start_delta_ms = as_float(report.get("activity_start_delta_ms"), 0.0)
|
|
first_skew_ms = as_float(report.get("first_skew_ms"), 0.0)
|
|
activity_pair_disagreement_ms = (
|
|
activity_start_delta_ms - first_skew_ms
|
|
if as_int(report.get("paired_event_count"), 0) > 0
|
|
else 0.0
|
|
)
|
|
|
|
reasons = []
|
|
smoothness_warnings = []
|
|
if run_status != 0:
|
|
reasons.append(f"probe command exited {run_status}")
|
|
if as_bool(require_sync_raw) and not sync_pass:
|
|
reasons.append(f"sync did not pass: {verdict.get('status', 'unknown')}")
|
|
if as_bool(require_freshness_raw) and not freshness_pass:
|
|
reasons.append(f"freshness did not pass: {freshness_status}")
|
|
if capture_timebase and capture_timebase.get("valid") is False:
|
|
reasons.append(
|
|
"capture timebase invalid: "
|
|
f"{capture_timebase.get('reason', capture_timebase.get('status', 'invalid'))}"
|
|
)
|
|
if signature_expected > 0:
|
|
if require_all_coded_pairs and signature_paired < signature_expected:
|
|
reasons.append(f"paired coded signatures {signature_paired} < expected {signature_expected}")
|
|
elif min_coded_pairs > 0 and signature_paired < min_coded_pairs:
|
|
reasons.append(f"paired coded signatures {signature_paired} < required {min_coded_pairs}")
|
|
if signature_unknown > 0:
|
|
reasons.append(f"paired signatures without coded identity {signature_unknown} > 0")
|
|
elif report.get("coded_events") is True:
|
|
reasons.append("coded signature coverage unavailable")
|
|
if video_hiccups > as_int(max_video_hiccups_raw):
|
|
smoothness_warnings.append(f"video hiccups {video_hiccups} > {max_video_hiccups_raw}")
|
|
if audio_hiccups > as_int(max_audio_hiccups_raw):
|
|
smoothness_warnings.append(f"audio hiccups {audio_hiccups} > {max_audio_hiccups_raw}")
|
|
if video_jitter > as_float(max_video_jitter_raw):
|
|
smoothness_warnings.append(f"video p95 jitter {video_jitter:.1f}ms > {as_float(max_video_jitter_raw):.1f}ms")
|
|
if audio_jitter > as_float(max_audio_jitter_raw):
|
|
smoothness_warnings.append(f"audio p95 jitter {audio_jitter:.1f}ms > {as_float(max_audio_jitter_raw):.1f}ms")
|
|
if missing > as_int(max_missing_raw):
|
|
smoothness_warnings.append(f"estimated missing video frames {missing} > {max_missing_raw}")
|
|
if undecodable > as_int(max_undecodable_raw):
|
|
smoothness_warnings.append(f"undecodable video frames {undecodable} > {max_undecodable_raw}")
|
|
if duplicates > as_int(max_duplicates_raw):
|
|
smoothness_warnings.append(f"duplicate video frames {duplicates} > {max_duplicates_raw}")
|
|
if low_rms > as_int(max_low_rms_raw):
|
|
smoothness_warnings.append(f"low-RMS audio windows {low_rms} > {max_low_rms_raw}")
|
|
if require_smoothness:
|
|
reasons.extend(smoothness_warnings)
|
|
|
|
artifact = {
|
|
"schema": "lesavka.server-rc-mode-result.v1",
|
|
"mode": mode,
|
|
"width": as_int(width_raw),
|
|
"height": as_int(height_raw),
|
|
"fps": as_int(fps_raw),
|
|
"audio_delay_us": as_int(audio_delay_raw),
|
|
"video_delay_us": as_int(video_delay_raw),
|
|
"run_status": run_status,
|
|
"run_log": run_log,
|
|
"artifact_dir": str(artifact_dir),
|
|
"report_json": str(artifact_dir / "report.json"),
|
|
"correlation_json": str(artifact_dir / "output-delay-correlation.json"),
|
|
"calibration_json": str(artifact_dir / "output-delay-calibration.json"),
|
|
"timing_map": timing_map,
|
|
"passed": not reasons,
|
|
"failure_reasons": reasons,
|
|
"smoothness_required": require_smoothness,
|
|
"smoothness_warnings": smoothness_warnings,
|
|
"event_visibility": event_visibility,
|
|
"sync": {
|
|
"passed": sync_pass,
|
|
"status": verdict.get("status", "unknown"),
|
|
"reason": verdict.get("reason", ""),
|
|
"p95_abs_skew_ms": as_float(verdict.get("p95_abs_skew_ms"), 0.0),
|
|
"median_skew_ms": as_float(report.get("median_skew_ms"), 0.0),
|
|
"drift_ms": as_float(report.get("drift_ms"), 0.0),
|
|
"activity_start_delta_ms": activity_start_delta_ms,
|
|
"first_skew_ms": first_skew_ms,
|
|
"activity_pair_disagreement_ms": activity_pair_disagreement_ms,
|
|
"video_event_count": as_int(report.get("video_event_count"), 0),
|
|
"audio_event_count": as_int(report.get("audio_event_count"), 0),
|
|
"paired_event_count": as_int(report.get("paired_event_count"), 0),
|
|
"signature_expected_event_count": signature_expected,
|
|
"signature_paired_event_count": signature_paired,
|
|
"signature_missing_event_ids": signature_coverage.get("missing_event_ids") or [],
|
|
"signature_missing_codes": signature_coverage.get("missing_codes") or [],
|
|
"signature_unknown_pair_identity_count": signature_unknown,
|
|
"paired_confidence_min": min(pair_confidences) if pair_confidences else 0.0,
|
|
"paired_confidence_median": median(pair_confidences),
|
|
},
|
|
"freshness": {
|
|
"status": freshness_status,
|
|
"reason": freshness.get("reason", ""),
|
|
"worst_event_age_p95_ms": freshness.get("worst_event_age_p95_ms"),
|
|
"worst_p95_pipeline_freshness_ms": freshness.get("worst_p95_pipeline_freshness_ms"),
|
|
"minimum_pipeline_freshness_ms": freshness.get("minimum_pipeline_freshness_ms"),
|
|
"clock_uncertainty_ms": freshness.get("clock_uncertainty_ms"),
|
|
"worst_event_age_with_uncertainty_ms": freshness.get("worst_event_age_with_uncertainty_ms"),
|
|
"worst_freshness_drift_ms": freshness.get("worst_freshness_drift_ms"),
|
|
"max_age_limit_ms": freshness.get("max_age_limit_ms"),
|
|
"capture_timebase_status": capture_timebase.get("status"),
|
|
"capture_timebase_valid": capture_timebase.get("valid"),
|
|
"capture_timebase_reason": capture_timebase.get("reason"),
|
|
},
|
|
"smoothness": {
|
|
"video_frames": as_int(video.get("timestamps"), 0),
|
|
"video_p95_jitter_ms": video_jitter,
|
|
"video_max_interval_ms": as_float(nested(video, "interval_stats", "max_interval_ms"), 0.0),
|
|
"video_hiccups": video_hiccups,
|
|
"video_decoded_frames": as_int(video.get("decoded_frames"), 0),
|
|
"video_duplicate_frames": duplicates,
|
|
"video_estimated_missing_frames": missing,
|
|
"video_undecodable_frames": undecodable,
|
|
"audio_packets": as_int(audio_cadence.get("timestamps"), 0),
|
|
"audio_p95_jitter_ms": audio_jitter,
|
|
"audio_max_interval_ms": as_float(nested(audio_cadence, "interval_stats", "max_interval_ms"), 0.0),
|
|
"audio_hiccups": audio_hiccups,
|
|
"audio_low_rms_windows": low_rms,
|
|
"audio_median_rms": as_float(nested(audio_rms, "rms_stats", "median_rms"), 0.0),
|
|
},
|
|
"output_delay_calibration": {
|
|
"ready": calibration.get("ready") is True,
|
|
"decision": calibration.get("decision", "unknown"),
|
|
"target": calibration.get("target", ""),
|
|
"paired_event_count": as_int(calibration.get("paired_event_count"), 0),
|
|
"min_pairs": as_int(calibration.get("min_pairs"), 0),
|
|
"measured_device_skew_ms": as_float(calibration.get("measured_device_skew_ms"), 0.0),
|
|
"p95_abs_skew_ms": as_float(calibration.get("p95_abs_skew_ms"), 0.0),
|
|
"max_abs_skew_ms": as_float(calibration.get("max_abs_skew_ms"), 0.0),
|
|
"drift_ms": as_float(calibration.get("drift_ms"), 0.0),
|
|
"active_audio_offset_us": as_int(calibration.get("active_audio_offset_us"), as_int(audio_delay_raw)),
|
|
"active_video_offset_us": as_int(calibration.get("active_video_offset_us"), as_int(video_delay_raw)),
|
|
"audio_offset_adjust_us": as_int(calibration.get("audio_offset_adjust_us"), 0),
|
|
"video_offset_adjust_us": as_int(calibration.get("video_offset_adjust_us"), 0),
|
|
"audio_target_offset_us": as_int(calibration.get("audio_target_offset_us"), 0),
|
|
"video_target_offset_us": as_int(calibration.get("video_target_offset_us"), 0),
|
|
"raw_device_delta_us": as_int(calibration.get("raw_device_delta_us"), 0),
|
|
"bounded_device_delta_us": as_int(calibration.get("bounded_device_delta_us"), 0),
|
|
"max_step_us": as_int(calibration.get("max_step_us"), 0),
|
|
"note": calibration.get("note", ""),
|
|
},
|
|
}
|
|
pathlib.Path(output_json).write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n")
|
|
PY
|
|
}
|
|
|
|
annotate_mode_result() {
|
|
local output_json=$1
|
|
local mode_id=$2
|
|
local mode_run_index=$3
|
|
local repeat_index=$4
|
|
local repeat_count=$5
|
|
local matrix_sequence=$6
|
|
local phase=$7
|
|
|
|
python3 - <<'PY' \
|
|
"${output_json}" \
|
|
"${mode_id}" \
|
|
"${mode_run_index}" \
|
|
"${repeat_index}" \
|
|
"${repeat_count}" \
|
|
"${matrix_sequence}" \
|
|
"${phase}"
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
(
|
|
output_json_raw,
|
|
mode_id,
|
|
mode_run_index_raw,
|
|
repeat_index_raw,
|
|
repeat_count_raw,
|
|
matrix_sequence_raw,
|
|
phase,
|
|
) = sys.argv[1:]
|
|
|
|
|
|
def as_int(value, default=0):
|
|
try:
|
|
return int(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
path = pathlib.Path(output_json_raw)
|
|
result = json.loads(path.read_text())
|
|
result.update(
|
|
{
|
|
"mode_id": mode_id,
|
|
"mode_run_index": as_int(mode_run_index_raw),
|
|
"repeat_index": as_int(repeat_index_raw),
|
|
"repeat_count": as_int(repeat_count_raw),
|
|
"matrix_sequence": as_int(matrix_sequence_raw),
|
|
"probe_phase": phase,
|
|
}
|
|
)
|
|
path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
|
PY
|
|
}
|
|
|
|
summarize_matrix() {
|
|
python3 - <<'PY' "${MATRIX_REPORT_DIR}" "${MATRIX_SUMMARY_JSON}" "${MATRIX_SUMMARY_CSV}" "${MATRIX_SUMMARY_TXT}" "${MATRIX_DELAY_JSON}" "${MATRIX_DELAY_ENV}" "${MATRIX_STATIC_JSON}" "${MATRIX_STATIC_CSV}" "${MATRIX_STATIC_TXT}" "${MATRIX_STATIC_ENV}" "${LESAVKA_SERVER_RC_STATIC_MIN_RUNS}" "${LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US}" "${LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS}" "${LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS}" "${LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS}" "${LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS}"
|
|
import csv
|
|
import json
|
|
import pathlib
|
|
import shlex
|
|
import sys
|
|
|
|
root = pathlib.Path(sys.argv[1])
|
|
summary_json = pathlib.Path(sys.argv[2])
|
|
summary_csv = pathlib.Path(sys.argv[3])
|
|
summary_txt = pathlib.Path(sys.argv[4])
|
|
delay_json = pathlib.Path(sys.argv[5])
|
|
delay_env = pathlib.Path(sys.argv[6])
|
|
static_json = pathlib.Path(sys.argv[7])
|
|
static_csv = pathlib.Path(sys.argv[8])
|
|
static_txt = pathlib.Path(sys.argv[9])
|
|
static_env = pathlib.Path(sys.argv[10])
|
|
static_min_runs = int(sys.argv[11])
|
|
static_max_spread_us = int(sys.argv[12])
|
|
static_max_p95_skew_ms = float(sys.argv[13])
|
|
static_max_median_skew_ms = float(sys.argv[14])
|
|
static_require_freshness = sys.argv[15].strip().lower() not in {"", "0", "false", "no", "off"}
|
|
static_require_smoothness = sys.argv[16].strip().lower() not in {"", "0", "false", "no", "off"}
|
|
|
|
|
|
def as_int(value, default=0):
|
|
try:
|
|
return int(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def as_float(value, default=0.0):
|
|
try:
|
|
return float(str(value).strip())
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def median(values, default=None):
|
|
cleaned = sorted(value for value in values if value is not None)
|
|
if not cleaned:
|
|
return default
|
|
mid = len(cleaned) // 2
|
|
if len(cleaned) % 2:
|
|
return cleaned[mid]
|
|
return (cleaned[mid - 1] + cleaned[mid]) / 2.0
|
|
|
|
|
|
def median_int(values, default=None):
|
|
value = median(values, default)
|
|
if value is None:
|
|
return None
|
|
return int(round(value))
|
|
|
|
|
|
results = []
|
|
for path in sorted(root.glob("*/mode-result.json")):
|
|
try:
|
|
result = json.loads(path.read_text())
|
|
seed_path = path.with_name("mode-result-seed.json")
|
|
tuned_path = path.with_name("mode-result-tuned.json")
|
|
if seed_path.exists():
|
|
result["seed_result_json"] = str(seed_path)
|
|
try:
|
|
seed = json.loads(seed_path.read_text())
|
|
result["seed_audio_delay_us"] = seed.get("audio_delay_us")
|
|
result["seed_video_delay_us"] = seed.get("video_delay_us")
|
|
except Exception:
|
|
pass
|
|
if tuned_path.exists():
|
|
result["tuned_result_json"] = str(tuned_path)
|
|
result["result_json"] = str(path)
|
|
results.append(result)
|
|
except Exception:
|
|
continue
|
|
results.sort(
|
|
key=lambda item: (
|
|
item.get("mode") or "",
|
|
as_int(item.get("mode_run_index"), as_int(item.get("matrix_sequence"), 0)),
|
|
as_int(item.get("matrix_sequence"), 0),
|
|
)
|
|
)
|
|
|
|
mode_groups = {}
|
|
for result in results:
|
|
mode = result.get("mode")
|
|
if not mode:
|
|
continue
|
|
mode_groups.setdefault(mode, []).append(result)
|
|
|
|
|
|
def static_eligibility(result):
|
|
sync = result.get("sync") or {}
|
|
freshness = result.get("freshness") or {}
|
|
calibration = result.get("output_delay_calibration") or {}
|
|
reasons = []
|
|
required_pairs = as_int(calibration.get("min_pairs"), 13) or 13
|
|
if result.get("passed") is not True:
|
|
reasons.append("probe did not pass all required gates")
|
|
if sync.get("passed") is not True:
|
|
reasons.append(f"sync did not pass ({sync.get('status', 'unknown')})")
|
|
if as_int(sync.get("paired_event_count"), 0) < required_pairs:
|
|
reasons.append(f"paired events {sync.get('paired_event_count', 0)} < {required_pairs}")
|
|
if as_float(sync.get("p95_abs_skew_ms"), 0.0) > static_max_p95_skew_ms:
|
|
reasons.append(
|
|
f"p95 skew {as_float(sync.get('p95_abs_skew_ms'), 0.0):.1f}ms > {static_max_p95_skew_ms:.1f}ms"
|
|
)
|
|
if abs(as_float(sync.get("median_skew_ms"), 0.0)) > static_max_median_skew_ms:
|
|
reasons.append(
|
|
f"median skew {as_float(sync.get('median_skew_ms'), 0.0):+.1f}ms outside +/-{static_max_median_skew_ms:.1f}ms"
|
|
)
|
|
if static_require_freshness and freshness.get("status") != "pass":
|
|
reasons.append(f"freshness did not pass ({freshness.get('status', 'unknown')})")
|
|
if static_require_smoothness and result.get("smoothness_warnings"):
|
|
reasons.append("smoothness warnings present")
|
|
if not isinstance(result.get("video_delay_us"), int) or not isinstance(result.get("audio_delay_us"), int):
|
|
reasons.append("tested delay values unavailable")
|
|
return reasons
|
|
|
|
|
|
static_modes = {}
|
|
delay_recommendations = {}
|
|
video_delay_entries = []
|
|
audio_delay_entries = []
|
|
static_video_entries = []
|
|
static_audio_entries = []
|
|
for mode, mode_results in sorted(mode_groups.items()):
|
|
eligible = []
|
|
rejected = []
|
|
for result in mode_results:
|
|
reasons = static_eligibility(result)
|
|
if reasons:
|
|
rejected.append(
|
|
{
|
|
"mode_run_index": result.get("mode_run_index"),
|
|
"artifact_dir": result.get("artifact_dir"),
|
|
"reasons": reasons,
|
|
"tested_video_delay_us": result.get("video_delay_us"),
|
|
"tested_audio_delay_us": result.get("audio_delay_us"),
|
|
"p95_abs_skew_ms": (result.get("sync") or {}).get("p95_abs_skew_ms"),
|
|
"median_skew_ms": (result.get("sync") or {}).get("median_skew_ms"),
|
|
}
|
|
)
|
|
else:
|
|
eligible.append(result)
|
|
|
|
tested_video = [result.get("video_delay_us") for result in eligible]
|
|
tested_audio = [result.get("audio_delay_us") for result in eligible]
|
|
target_video = [
|
|
(result.get("output_delay_calibration") or {}).get("video_target_offset_us")
|
|
for result in eligible
|
|
if isinstance((result.get("output_delay_calibration") or {}).get("video_target_offset_us"), int)
|
|
]
|
|
target_audio = [
|
|
(result.get("output_delay_calibration") or {}).get("audio_target_offset_us")
|
|
for result in eligible
|
|
if isinstance((result.get("output_delay_calibration") or {}).get("audio_target_offset_us"), int)
|
|
]
|
|
tested_video_spread = (max(tested_video) - min(tested_video)) if tested_video else None
|
|
tested_audio_spread = (max(tested_audio) - min(tested_audio)) if tested_audio else None
|
|
target_video_spread = (max(target_video) - min(target_video)) if target_video else None
|
|
recommended_video = median_int(tested_video)
|
|
recommended_audio = median_int(tested_audio)
|
|
status_reasons = []
|
|
if len(eligible) < static_min_runs:
|
|
status_reasons.append(f"eligible runs {len(eligible)} < {static_min_runs}")
|
|
if tested_video_spread is not None and tested_video_spread > static_max_spread_us:
|
|
status_reasons.append(f"tested video delay spread {tested_video_spread}us > {static_max_spread_us}us")
|
|
if tested_audio_spread is not None and tested_audio_spread > static_max_spread_us:
|
|
status_reasons.append(f"tested audio delay spread {tested_audio_spread}us > {static_max_spread_us}us")
|
|
if recommended_video is None or recommended_audio is None:
|
|
status_reasons.append("no recommended delay could be derived")
|
|
if len(eligible) < static_min_runs:
|
|
status = "needs_more_runs"
|
|
elif status_reasons:
|
|
status = "unstable"
|
|
else:
|
|
status = "ready"
|
|
|
|
p95_values = [as_float((result.get("sync") or {}).get("p95_abs_skew_ms"), 0.0) for result in eligible]
|
|
median_skews = [as_float((result.get("sync") or {}).get("median_skew_ms"), 0.0) for result in eligible]
|
|
freshness_budgets = [
|
|
as_float((result.get("freshness") or {}).get("worst_event_age_with_uncertainty_ms"), 0.0)
|
|
for result in eligible
|
|
]
|
|
static_entry = {
|
|
"status": status,
|
|
"ready": status == "ready",
|
|
"reasons": status_reasons,
|
|
"mode": mode,
|
|
"total_runs": len(mode_results),
|
|
"eligible_runs": len(eligible),
|
|
"rejected_runs": rejected,
|
|
"recommended_video_delay_us": recommended_video,
|
|
"recommended_audio_delay_us": recommended_audio,
|
|
"tested_video_delay_us": tested_video,
|
|
"tested_audio_delay_us": tested_audio,
|
|
"tested_video_delay_min_us": min(tested_video) if tested_video else None,
|
|
"tested_video_delay_max_us": max(tested_video) if tested_video else None,
|
|
"tested_video_delay_spread_us": tested_video_spread,
|
|
"tested_audio_delay_spread_us": tested_audio_spread,
|
|
"target_video_delay_us": target_video,
|
|
"target_audio_delay_us": target_audio,
|
|
"target_video_delay_median_us": median_int(target_video),
|
|
"target_audio_delay_median_us": median_int(target_audio),
|
|
"target_video_delay_spread_us": target_video_spread,
|
|
"sync_p95_abs_skew_ms_max": max(p95_values) if p95_values else None,
|
|
"sync_p95_abs_skew_ms_median": median(p95_values),
|
|
"sync_abs_median_skew_ms_max": max((abs(value) for value in median_skews), default=None),
|
|
"freshness_budget_ms_max": max(freshness_budgets) if freshness_budgets else None,
|
|
"eligible_artifact_dirs": [result.get("artifact_dir") for result in eligible],
|
|
"eligible_result_json": [result.get("result_json") for result in eligible],
|
|
}
|
|
static_modes[mode] = static_entry
|
|
|
|
if status == "ready" and isinstance(recommended_video, int) and isinstance(recommended_audio, int):
|
|
static_video_entries.append(f"{mode}={recommended_video}")
|
|
static_audio_entries.append(f"{mode}={recommended_audio}")
|
|
|
|
delay_video = recommended_video
|
|
delay_audio = recommended_audio
|
|
recommendation_status = "static_ready" if status == "ready" else status
|
|
if status != "ready":
|
|
fallback = (eligible or mode_results)[-1]
|
|
fallback_calibration = fallback.get("output_delay_calibration") or {}
|
|
fallback_video_target = fallback_calibration.get("video_target_offset_us")
|
|
fallback_audio_target = fallback_calibration.get("audio_target_offset_us")
|
|
if not isinstance(delay_video, int) or not isinstance(delay_audio, int):
|
|
if (
|
|
fallback_calibration.get("ready") is True
|
|
and isinstance(fallback_video_target, int)
|
|
and isinstance(fallback_audio_target, int)
|
|
):
|
|
delay_video = fallback_video_target
|
|
delay_audio = fallback_audio_target
|
|
recommendation_status = "candidate_unconfirmed"
|
|
else:
|
|
delay_video = fallback.get("video_delay_us")
|
|
delay_audio = fallback.get("audio_delay_us")
|
|
recommendation_status = "tested_unstable"
|
|
else:
|
|
recommendation_status = f"candidate_{status}"
|
|
if isinstance(delay_video, int) and isinstance(delay_audio, int):
|
|
video_delay_entries.append(f"{mode}={delay_video}")
|
|
audio_delay_entries.append(f"{mode}={delay_audio}")
|
|
delay_recommendations[mode] = {
|
|
"status": recommendation_status,
|
|
"static_status": status,
|
|
"audio_delay_us": delay_audio,
|
|
"video_delay_us": delay_video,
|
|
"eligible_runs": len(eligible),
|
|
"total_runs": len(mode_results),
|
|
"tested_video_delay_spread_us": tested_video_spread,
|
|
"sync_p95_abs_skew_ms_max": static_entry["sync_p95_abs_skew_ms_max"],
|
|
"sync_abs_median_skew_ms_max": static_entry["sync_abs_median_skew_ms_max"],
|
|
"freshness_budget_ms_max": static_entry["freshness_budget_ms_max"],
|
|
"reasons": status_reasons,
|
|
}
|
|
|
|
summary = {
|
|
"schema": "lesavka.server-rc-mode-matrix-summary.v1",
|
|
"artifact_dir": str(root),
|
|
"passed": bool(results) and all(result.get("passed") for result in results),
|
|
"mode_count": len(results),
|
|
"delay_recommendations_json": str(delay_json),
|
|
"delay_recommendations_env": str(delay_env),
|
|
"static_calibration_json": str(static_json),
|
|
"static_calibration_csv": str(static_csv),
|
|
"static_calibration_txt": str(static_txt),
|
|
"static_calibration_env": str(static_env),
|
|
"static_ready": bool(static_modes) and all(entry.get("ready") for entry in static_modes.values()),
|
|
"static_calibration": static_modes,
|
|
"delay_recommendations": delay_recommendations,
|
|
"results": results,
|
|
}
|
|
summary_json.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
|
|
delay_json.write_text(
|
|
json.dumps(
|
|
{
|
|
"schema": "lesavka.server-rc-mode-delay-recommendations.v1",
|
|
"artifact_dir": str(root),
|
|
"video_delays_us": {
|
|
mode: entry.get("video_delay_us")
|
|
for mode, entry in delay_recommendations.items()
|
|
},
|
|
"audio_delays_us": {
|
|
mode: entry.get("audio_delay_us")
|
|
for mode, entry in delay_recommendations.items()
|
|
},
|
|
"recommendations": delay_recommendations,
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
+ "\n"
|
|
)
|
|
static_json.write_text(
|
|
json.dumps(
|
|
{
|
|
"schema": "lesavka.server-rc-static-calibration.v1",
|
|
"artifact_dir": str(root),
|
|
"ready": summary["static_ready"],
|
|
"criteria": {
|
|
"min_runs": static_min_runs,
|
|
"max_spread_us": static_max_spread_us,
|
|
"max_p95_skew_ms": static_max_p95_skew_ms,
|
|
"max_median_skew_ms": static_max_median_skew_ms,
|
|
"require_freshness": static_require_freshness,
|
|
"require_smoothness": static_require_smoothness,
|
|
},
|
|
"video_delays_us": {
|
|
mode: entry.get("recommended_video_delay_us")
|
|
for mode, entry in static_modes.items()
|
|
if entry.get("ready")
|
|
},
|
|
"audio_delays_us": {
|
|
mode: entry.get("recommended_audio_delay_us")
|
|
for mode, entry in static_modes.items()
|
|
if entry.get("ready")
|
|
},
|
|
"modes": static_modes,
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
)
|
|
+ "\n"
|
|
)
|
|
delay_env.write_text(
|
|
"LESAVKA_SERVER_RC_MODE_DELAYS_US="
|
|
+ shlex.quote(",".join(video_delay_entries))
|
|
+ "\n"
|
|
+ "LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US="
|
|
+ shlex.quote(",".join(audio_delay_entries))
|
|
+ "\n"
|
|
)
|
|
static_env.write_text(
|
|
"LESAVKA_SERVER_RC_MODE_DELAYS_US="
|
|
+ shlex.quote(",".join(static_video_entries))
|
|
+ "\n"
|
|
+ "LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US="
|
|
+ shlex.quote(",".join(static_audio_entries))
|
|
+ "\n"
|
|
)
|
|
|
|
fieldnames = [
|
|
"mode",
|
|
"mode_run_index",
|
|
"repeat_index",
|
|
"matrix_sequence",
|
|
"passed",
|
|
"seed_video_delay_us",
|
|
"seed_audio_delay_us",
|
|
"video_delay_us",
|
|
"audio_delay_us",
|
|
"sync_status",
|
|
"p95_abs_skew_ms",
|
|
"median_skew_ms",
|
|
"sync_drift_ms",
|
|
"freshness_status",
|
|
"freshness_budget_ms",
|
|
"freshness_drift_ms",
|
|
"capture_timebase_status",
|
|
"video_hiccups",
|
|
"video_missing",
|
|
"video_undecodable",
|
|
"audio_hiccups",
|
|
"audio_low_rms",
|
|
"calibration_ready",
|
|
"calibration_decision",
|
|
"calibration_target",
|
|
"calibration_measured_skew_ms",
|
|
"calibration_video_target_offset_us",
|
|
"calibration_audio_target_offset_us",
|
|
"artifact_dir",
|
|
"result_json",
|
|
]
|
|
with summary_csv.open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
for result in results:
|
|
writer.writerow({
|
|
"mode": result.get("mode"),
|
|
"mode_run_index": result.get("mode_run_index"),
|
|
"repeat_index": result.get("repeat_index"),
|
|
"matrix_sequence": result.get("matrix_sequence"),
|
|
"passed": result.get("passed"),
|
|
"seed_video_delay_us": result.get("seed_video_delay_us"),
|
|
"seed_audio_delay_us": result.get("seed_audio_delay_us"),
|
|
"video_delay_us": result.get("video_delay_us"),
|
|
"audio_delay_us": result.get("audio_delay_us"),
|
|
"sync_status": (result.get("sync") or {}).get("status"),
|
|
"p95_abs_skew_ms": (result.get("sync") or {}).get("p95_abs_skew_ms"),
|
|
"median_skew_ms": (result.get("sync") or {}).get("median_skew_ms"),
|
|
"sync_drift_ms": (result.get("sync") or {}).get("drift_ms"),
|
|
"freshness_status": (result.get("freshness") or {}).get("status"),
|
|
"freshness_budget_ms": (result.get("freshness") or {}).get("worst_event_age_with_uncertainty_ms"),
|
|
"freshness_drift_ms": (result.get("freshness") or {}).get("worst_freshness_drift_ms"),
|
|
"capture_timebase_status": (result.get("freshness") or {}).get("capture_timebase_status"),
|
|
"video_hiccups": (result.get("smoothness") or {}).get("video_hiccups"),
|
|
"video_missing": (result.get("smoothness") or {}).get("video_estimated_missing_frames"),
|
|
"video_undecodable": (result.get("smoothness") or {}).get("video_undecodable_frames"),
|
|
"audio_hiccups": (result.get("smoothness") or {}).get("audio_hiccups"),
|
|
"audio_low_rms": (result.get("smoothness") or {}).get("audio_low_rms_windows"),
|
|
"calibration_ready": (result.get("output_delay_calibration") or {}).get("ready"),
|
|
"calibration_decision": (result.get("output_delay_calibration") or {}).get("decision"),
|
|
"calibration_target": (result.get("output_delay_calibration") or {}).get("target"),
|
|
"calibration_measured_skew_ms": (result.get("output_delay_calibration") or {}).get("measured_device_skew_ms"),
|
|
"calibration_video_target_offset_us": (result.get("output_delay_calibration") or {}).get("video_target_offset_us"),
|
|
"calibration_audio_target_offset_us": (result.get("output_delay_calibration") or {}).get("audio_target_offset_us"),
|
|
"artifact_dir": result.get("artifact_dir"),
|
|
"result_json": result.get("result_json"),
|
|
})
|
|
|
|
static_fieldnames = [
|
|
"mode",
|
|
"status",
|
|
"ready",
|
|
"eligible_runs",
|
|
"total_runs",
|
|
"recommended_video_delay_us",
|
|
"recommended_audio_delay_us",
|
|
"tested_video_delay_min_us",
|
|
"tested_video_delay_max_us",
|
|
"tested_video_delay_spread_us",
|
|
"target_video_delay_median_us",
|
|
"target_video_delay_spread_us",
|
|
"sync_p95_abs_skew_ms_max",
|
|
"sync_p95_abs_skew_ms_median",
|
|
"sync_abs_median_skew_ms_max",
|
|
"freshness_budget_ms_max",
|
|
"reasons",
|
|
]
|
|
with static_csv.open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=static_fieldnames)
|
|
writer.writeheader()
|
|
for mode, entry in sorted(static_modes.items()):
|
|
writer.writerow(
|
|
{
|
|
"mode": mode,
|
|
"status": entry.get("status"),
|
|
"ready": entry.get("ready"),
|
|
"eligible_runs": entry.get("eligible_runs"),
|
|
"total_runs": entry.get("total_runs"),
|
|
"recommended_video_delay_us": entry.get("recommended_video_delay_us"),
|
|
"recommended_audio_delay_us": entry.get("recommended_audio_delay_us"),
|
|
"tested_video_delay_min_us": entry.get("tested_video_delay_min_us"),
|
|
"tested_video_delay_max_us": entry.get("tested_video_delay_max_us"),
|
|
"tested_video_delay_spread_us": entry.get("tested_video_delay_spread_us"),
|
|
"target_video_delay_median_us": entry.get("target_video_delay_median_us"),
|
|
"target_video_delay_spread_us": entry.get("target_video_delay_spread_us"),
|
|
"sync_p95_abs_skew_ms_max": entry.get("sync_p95_abs_skew_ms_max"),
|
|
"sync_p95_abs_skew_ms_median": entry.get("sync_p95_abs_skew_ms_median"),
|
|
"sync_abs_median_skew_ms_max": entry.get("sync_abs_median_skew_ms_max"),
|
|
"freshness_budget_ms_max": entry.get("freshness_budget_ms_max"),
|
|
"reasons": "; ".join(entry.get("reasons") or []),
|
|
}
|
|
)
|
|
|
|
lines = [
|
|
f"Server-to-RC mode matrix for {root}",
|
|
f"- runs: {len(results)}",
|
|
f"- unique modes: {len(mode_groups)}",
|
|
f"- verdict: {'pass' if summary['passed'] else 'fail'}",
|
|
f"- static calibration: {'ready' if summary['static_ready'] else 'needs-more-data'}",
|
|
]
|
|
for result in results:
|
|
sync = result.get("sync") or {}
|
|
freshness = result.get("freshness") or {}
|
|
smooth = result.get("smoothness") or {}
|
|
calibration = result.get("output_delay_calibration") or {}
|
|
marker = "PASS" if result.get("passed") else "FAIL"
|
|
lines.append(
|
|
f"- {marker} {result.get('mode')} run={result.get('mode_run_index', '?')}: "
|
|
f"delays video={result.get('video_delay_us', 0)}us audio={result.get('audio_delay_us', 0)}us; "
|
|
f"sync {sync.get('status')} p95={sync.get('p95_abs_skew_ms', 0.0):.1f}ms median={sync.get('median_skew_ms', 0.0):+.1f}ms; "
|
|
f"freshness {freshness.get('status')} budget={freshness.get('worst_event_age_with_uncertainty_ms') or 0.0:.1f}ms; "
|
|
f"smooth video hiccups={smooth.get('video_hiccups', 0)} missing={smooth.get('video_estimated_missing_frames', 0)} undecodable={smooth.get('video_undecodable_frames', 0)} audio hiccups={smooth.get('audio_hiccups', 0)}"
|
|
)
|
|
lines.append(
|
|
" calibration: "
|
|
f"{calibration.get('decision', 'unknown')} ready={calibration.get('ready', False)} "
|
|
f"target={calibration.get('target', '')} measured={calibration.get('measured_device_skew_ms', 0.0):+.1f}ms "
|
|
f"video_target={calibration.get('video_target_offset_us', 0)}us "
|
|
f"audio_target={calibration.get('audio_target_offset_us', 0)}us"
|
|
)
|
|
lines.append(
|
|
" sync evidence: "
|
|
f"video_onsets={sync.get('video_event_count', 0)} audio_onsets={sync.get('audio_event_count', 0)} "
|
|
f"pairs={sync.get('paired_event_count', 0)} "
|
|
f"coded_pairs={sync.get('signature_paired_event_count', 0)}/{sync.get('signature_expected_event_count', 0)} "
|
|
f"missing_codes={sync.get('signature_missing_codes') or []} "
|
|
f"unknown_identity={sync.get('signature_unknown_pair_identity_count', 0)} "
|
|
f"pair_conf_median={sync.get('paired_confidence_median', 0.0):.3f} "
|
|
f"raw_pair_disagreement={sync.get('activity_pair_disagreement_ms', 0.0):+.1f}ms"
|
|
)
|
|
visibility = result.get("event_visibility") or {}
|
|
visibility_summary = visibility.get("summary") or {}
|
|
if visibility_summary:
|
|
lines.append(
|
|
" coded visibility: "
|
|
f"paired={visibility_summary.get('paired', 0)} "
|
|
f"video_only={visibility_summary.get('video_only', 0)} "
|
|
f"audio_only={visibility_summary.get('audio_only', 0)} "
|
|
f"unpaired={visibility_summary.get('unpaired', 0)} "
|
|
f"missing={visibility_summary.get('missing', 0)}"
|
|
)
|
|
if freshness.get("capture_timebase_status"):
|
|
lines.append(
|
|
" capture timing: "
|
|
f"{freshness.get('capture_timebase_status')} "
|
|
f"valid={freshness.get('capture_timebase_valid')} "
|
|
f"reason={freshness.get('capture_timebase_reason', '')}"
|
|
)
|
|
signal_readiness = result.get("signal_readiness") or {}
|
|
signal_attempts = signal_readiness.get("attempts") or []
|
|
if signal_readiness:
|
|
lines.append(
|
|
" signal readiness: "
|
|
f"passed={signal_readiness.get('passed', False)} "
|
|
f"attempts={len(signal_attempts)} "
|
|
f"reason={signal_readiness.get('reason', '')}"
|
|
)
|
|
for attempt in signal_attempts:
|
|
counts = attempt.get("counts") or {}
|
|
layers = attempt.get("layers") or {}
|
|
lines.append(
|
|
" signal attempt "
|
|
f"{attempt.get('attempt', '?')}: passed={attempt.get('passed', False)} "
|
|
f"server_events={counts.get('server_events', 0)} "
|
|
f"video={counts.get('video_events', 0)} audio={counts.get('audio_events', 0)} "
|
|
f"paired={counts.get('paired_events', counts.get('coded_pairs', 0))} "
|
|
f"audio_ready={layers.get('tethys_audio_ready', False)} "
|
|
f"video_ready={layers.get('tethys_video_ready', False)} "
|
|
f"reason={attempt.get('reason', '')}"
|
|
)
|
|
if "seed_video_delay_us" in result or "seed_audio_delay_us" in result:
|
|
lines.append(
|
|
" tuning: "
|
|
f"seed video={result.get('seed_video_delay_us', 0)}us audio={result.get('seed_audio_delay_us', 0)}us -> "
|
|
f"tested video={result.get('video_delay_us', 0)}us audio={result.get('audio_delay_us', 0)}us"
|
|
)
|
|
for reason in result.get("failure_reasons") or []:
|
|
lines.append(f" reason: {reason}")
|
|
for warning in result.get("smoothness_warnings") or []:
|
|
lines.append(f" smoothness warning: {warning}")
|
|
if video_delay_entries or audio_delay_entries:
|
|
lines.append(f"- recommended video delays: {','.join(video_delay_entries)}")
|
|
lines.append(f"- recommended audio delays: {','.join(audio_delay_entries)}")
|
|
static_lines = [
|
|
f"Server-to-RC static calibration for {root}",
|
|
f"- verdict: {'ready' if summary['static_ready'] else 'needs-more-data'}",
|
|
(
|
|
"- criteria: "
|
|
f"min_runs={static_min_runs} "
|
|
f"max_spread_us={static_max_spread_us} "
|
|
f"max_p95_skew_ms={static_max_p95_skew_ms:.1f} "
|
|
f"max_median_skew_ms={static_max_median_skew_ms:.1f} "
|
|
f"require_freshness={static_require_freshness} "
|
|
f"require_smoothness={static_require_smoothness}"
|
|
),
|
|
]
|
|
for mode, entry in sorted(static_modes.items()):
|
|
marker = "READY" if entry.get("ready") else "HOLD"
|
|
reasons = "; ".join(entry.get("reasons") or [])
|
|
if not reasons and entry.get("ready"):
|
|
reasons = "all stability criteria met"
|
|
static_lines.append(
|
|
f"- {marker} {mode}: "
|
|
f"runs={entry.get('eligible_runs')}/{entry.get('total_runs')} "
|
|
f"video={entry.get('recommended_video_delay_us')}us "
|
|
f"audio={entry.get('recommended_audio_delay_us')}us "
|
|
f"spread={entry.get('tested_video_delay_spread_us')}us "
|
|
f"p95_max={entry.get('sync_p95_abs_skew_ms_max') or 0.0:.1f}ms "
|
|
f"median_abs_max={entry.get('sync_abs_median_skew_ms_max') or 0.0:.1f}ms "
|
|
f"freshness_budget_max={entry.get('freshness_budget_ms_max') or 0.0:.1f}ms"
|
|
)
|
|
static_lines.append(f" reason: {reasons}")
|
|
if static_video_entries or static_audio_entries:
|
|
static_lines.append(f"- static video delays: {','.join(static_video_entries)}")
|
|
static_lines.append(f"- static audio delays: {','.join(static_audio_entries)}")
|
|
lines.extend(["", *static_lines])
|
|
summary_txt.write_text("\n".join(lines) + "\n")
|
|
static_txt.write_text("\n".join(static_lines) + "\n")
|
|
print("\n".join(lines))
|
|
PY
|
|
}
|
|
|
|
if [[ "${LESAVKA_SERVER_RC_MODES}" == "auto" ]]; then
|
|
echo "==> discovering local webcam-backed Lesavka modes"
|
|
LESAVKA_SERVER_RC_MODES="$(discover_local_webcam_modes)"
|
|
LESAVKA_SERVER_RC_MODE_SOURCE=local-v4l2
|
|
fi
|
|
|
|
echo "==> server-to-RC mode matrix"
|
|
echo " ↪ modes=${LESAVKA_SERVER_RC_MODES}"
|
|
echo " ↪ mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}"
|
|
echo " ↪ repeat_count=${LESAVKA_SERVER_RC_REPEAT_COUNT} verbose_probes=${LESAVKA_SERVER_RC_VERBOSE_PROBES}"
|
|
echo " ↪ video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}"
|
|
echo " ↪ audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}"
|
|
echo " ↪ capture_stack=${REMOTE_CAPTURE_STACK} audio_source=${REMOTE_AUDIO_SOURCE} pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL} video_mode=${REMOTE_PULSE_VIDEO_MODE}"
|
|
echo " ↪ tune_delays=${LESAVKA_SERVER_RC_TUNE_DELAYS} confirm=${LESAVKA_SERVER_RC_TUNE_CONFIRM} min_pairs=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS} max_abs_skew_ms=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS} max_step_us=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US} min_change_us=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}"
|
|
echo " ↪ static_min_runs=${LESAVKA_SERVER_RC_STATIC_MIN_RUNS} static_max_spread_us=${LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US} static_max_p95_skew_ms=${LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS} static_max_median_skew_ms=${LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS}"
|
|
echo " ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS} min_pairs=${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}"
|
|
echo " ↪ coded_pairs_min=${LESAVKA_SERVER_RC_MIN_CODED_PAIRS} require_all_coded=${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS} smoothness_gate=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS}"
|
|
echo " ↪ signal_ready=${LESAVKA_SERVER_RC_SIGNAL_READY} mode=${LESAVKA_SERVER_RC_SIGNAL_READY_MODE} attempts=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS} min_pairs=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} duration=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}s warmup=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}s retry_delay=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s"
|
|
echo " ↪ signal_conditioning=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS}s warmup=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS}s gap=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS}s analysis_window=${LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW}"
|
|
echo " ↪ reconfigure=${LESAVKA_SERVER_RC_RECONFIGURE} strategy=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY} allow_gadget_reset=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET}"
|
|
echo " ↪ tethys_ready=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY} settle=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}s timeout=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}s preroll_discard=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}s"
|
|
echo " ↪ start_delay=${LESAVKA_SERVER_RC_START_DELAY_SECONDS}s"
|
|
echo " ↪ artifact_dir=${MATRIX_REPORT_DIR}"
|
|
echo " ↪ matrix_run_log=${MATRIX_RUN_LOG}"
|
|
|
|
prime_remote_sudo
|
|
sleep_start_delay
|
|
prebuild_probe_tools
|
|
|
|
if ! [[ "${LESAVKA_SERVER_RC_REPEAT_COUNT}" =~ ^[0-9]+$ ]] || (( LESAVKA_SERVER_RC_REPEAT_COUNT < 1 )); then
|
|
printf 'LESAVKA_SERVER_RC_REPEAT_COUNT must be a positive integer; got %s\n' "${LESAVKA_SERVER_RC_REPEAT_COUNT}" >&2
|
|
exit 64
|
|
fi
|
|
|
|
declare -A mode_run_counts=()
|
|
matrix_sequence=0
|
|
IFS=',' read -r -a modes <<<"${LESAVKA_SERVER_RC_MODES}"
|
|
for mode in "${modes[@]}"; do
|
|
mode="${mode//[[:space:]]/}"
|
|
[[ -n "${mode}" ]] || continue
|
|
read -r width height fps < <(parse_mode "${mode}")
|
|
|
|
for repeat_index in $(seq 1 "${LESAVKA_SERVER_RC_REPEAT_COUNT}"); do
|
|
mode_run_counts["${mode}"]=$(( ${mode_run_counts["${mode}"]:-0} + 1 ))
|
|
mode_run_index=${mode_run_counts["${mode}"]}
|
|
matrix_sequence=$((matrix_sequence + 1))
|
|
video_delay_us="$(lookup_video_delay_us "${mode}")"
|
|
audio_delay_us="$(lookup_audio_delay_us "${mode}")"
|
|
id_base="$(mode_id "${mode}")"
|
|
run_label="$(printf '%02d' "${mode_run_index}")"
|
|
id="${id_base}__run${run_label}"
|
|
mode_dir="${MATRIX_REPORT_DIR}/${id}"
|
|
mode_log="${mode_dir}/mode-run.log"
|
|
mode_result="${mode_dir}/mode-result.json"
|
|
seed_result="${mode_dir}/mode-result-seed.json"
|
|
readiness_dir="${mode_dir}/signal-readiness"
|
|
readiness_log="${readiness_dir}/signal-readiness-run.log"
|
|
readiness_attempts_json="${readiness_dir}/signal-readiness-attempts.json"
|
|
tuned_log="${mode_dir}/mode-tuned-run.log"
|
|
tuned_result="${mode_dir}/mode-result-tuned.json"
|
|
tune_env="${mode_dir}/mode-tune-candidate.env"
|
|
mkdir -p "${mode_dir}"
|
|
|
|
echo "==> mode ${mode} run ${mode_run_index} repeat ${repeat_index}/${LESAVKA_SERVER_RC_REPEAT_COUNT}: video_delay_us=${video_delay_us} audio_delay_us=${audio_delay_us}"
|
|
reconfigure_server_mode "${mode}" "${width}" "${height}" "${fps}" "${audio_delay_us}" "${video_delay_us}"
|
|
wait_tethys_media_ready "${mode}" "${width}" "${height}" "${fps}"
|
|
|
|
if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" && "${LESAVKA_SERVER_RC_SIGNAL_READY_MODE}" != "separate" ]]; then
|
|
echo "==> mode ${mode} run ${mode_run_index}: using same-capture signal conditioning before measured probe"
|
|
fi
|
|
|
|
if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" && "${LESAVKA_SERVER_RC_SIGNAL_READY_MODE}" == "separate" ]]; then
|
|
mkdir -p "${readiness_dir}"
|
|
echo "==> mode ${mode} run ${mode_run_index}: proving Tethys signal readiness before measured probe"
|
|
readiness_pass=1
|
|
readiness_status=0
|
|
readiness_artifact_dir=""
|
|
readiness_reason="no readiness attempts ran"
|
|
readiness_attempt_jsons=()
|
|
for readiness_attempt in $(seq 1 "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}"); do
|
|
attempt_dir="${readiness_dir}/attempt-${readiness_attempt}"
|
|
readiness_log="${attempt_dir}/signal-readiness-run.log"
|
|
readiness_attempt_json="${attempt_dir}/signal-readiness-attempt.json"
|
|
mkdir -p "${attempt_dir}"
|
|
echo " ↪ readiness attempt ${readiness_attempt}/${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}: requiring ${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} paired coded events"
|
|
run_mode_probe \
|
|
"${width}" \
|
|
"${height}" \
|
|
"${fps}" \
|
|
"${audio_delay_us}" \
|
|
"${video_delay_us}" \
|
|
"${attempt_dir}" \
|
|
"${readiness_log}" \
|
|
"${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}" \
|
|
"${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}" \
|
|
"${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \
|
|
0
|
|
readiness_status=${RUN_MODE_PROBE_STATUS}
|
|
readiness_artifact_dir="$(artifact_dir_from_log "${readiness_log}" "${attempt_dir}")"
|
|
set +e
|
|
readiness_reason="$(
|
|
write_signal_readiness_attempt_result \
|
|
"${readiness_attempt}" \
|
|
"${readiness_artifact_dir}" \
|
|
"${readiness_status}" \
|
|
"${readiness_log}" \
|
|
"${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \
|
|
"${readiness_attempt_json}" 2>&1
|
|
)"
|
|
readiness_pass=$?
|
|
set -e
|
|
readiness_attempt_jsons+=("${readiness_attempt_json}")
|
|
if [[ "${readiness_pass}" -eq 0 ]]; then
|
|
echo " ↪ readiness attempt ${readiness_attempt} passed: ${readiness_reason}"
|
|
break
|
|
fi
|
|
[[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed"
|
|
echo " ↪ readiness attempt ${readiness_attempt} failed: ${readiness_reason}"
|
|
if [[ "${readiness_attempt}" -lt "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}" ]]; then
|
|
echo " ↪ waiting ${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s before retrying signal readiness"
|
|
sleep "${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}"
|
|
fi
|
|
done
|
|
write_signal_readiness_attempts_summary "${readiness_attempts_json}" "${readiness_attempt_jsons[@]}"
|
|
if [[ "${readiness_pass}" -ne 0 ]]; then
|
|
[[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed"
|
|
echo " ↪ signal readiness failed: ${readiness_reason}"
|
|
write_signal_readiness_failure \
|
|
"${mode}" \
|
|
"${width}" \
|
|
"${height}" \
|
|
"${fps}" \
|
|
"${video_delay_us}" \
|
|
"${audio_delay_us}" \
|
|
"${readiness_status}" \
|
|
"${readiness_log}" \
|
|
"${readiness_artifact_dir}" \
|
|
"${readiness_attempts_json}" \
|
|
"${readiness_reason}" \
|
|
"${mode_result}"
|
|
annotate_mode_result "${mode_result}" "${id_base}" "${mode_run_index}" "${repeat_index}" "${LESAVKA_SERVER_RC_REPEAT_COUNT}" "${matrix_sequence}" "readiness-failure"
|
|
cp "${mode_result}" "${seed_result}"
|
|
if [[ "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then
|
|
break 2
|
|
fi
|
|
continue
|
|
fi
|
|
echo " ↪ signal readiness passed: artifact_dir=${readiness_artifact_dir}"
|
|
fi
|
|
|
|
echo "==> mode ${mode} run ${mode_run_index}: running seed probe"
|
|
run_mode_probe "${width}" "${height}" "${fps}" "${audio_delay_us}" "${video_delay_us}" "${mode_dir}" "${mode_log}"
|
|
run_status=${RUN_MODE_PROBE_STATUS}
|
|
artifact_dir="$(artifact_dir_from_log "${mode_log}" "${mode_dir}")"
|
|
write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${video_delay_us}" "${audio_delay_us}" "${run_status}" "${mode_log}" "${artifact_dir}" "${mode_result}"
|
|
annotate_mode_result "${mode_result}" "${id_base}" "${mode_run_index}" "${repeat_index}" "${LESAVKA_SERVER_RC_REPEAT_COUNT}" "${matrix_sequence}" "seed"
|
|
cp "${mode_result}" "${seed_result}"
|
|
|
|
if [[ "${LESAVKA_SERVER_RC_TUNE_DELAYS}" != "0" && "${LESAVKA_SERVER_RC_TUNE_CONFIRM}" != "0" ]]; then
|
|
write_tune_candidate_env "${seed_result}" "${tune_env}"
|
|
# shellcheck disable=SC1090
|
|
source "${tune_env}"
|
|
if [[ "${tune_ready:-false}" == "true" ]]; then
|
|
echo "==> mode ${mode} run ${mode_run_index}: confirming tuned delays video_delay_us=${tune_video_delay_us} audio_delay_us=${tune_audio_delay_us}"
|
|
echo " ↪ tune delta: video=${tune_video_delta_us}us audio=${tune_audio_delta_us}us pairs=${tune_paired_event_count} drift=${tune_drift_ms}ms"
|
|
run_mode_probe "${width}" "${height}" "${fps}" "${tune_audio_delay_us}" "${tune_video_delay_us}" "${mode_dir}" "${tuned_log}"
|
|
tuned_status=${RUN_MODE_PROBE_STATUS}
|
|
tuned_artifact_dir="$(artifact_dir_from_log "${tuned_log}" "${mode_dir}")"
|
|
write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${tune_video_delay_us}" "${tune_audio_delay_us}" "${tuned_status}" "${tuned_log}" "${tuned_artifact_dir}" "${tuned_result}"
|
|
annotate_mode_result "${tuned_result}" "${id_base}" "${mode_run_index}" "${repeat_index}" "${LESAVKA_SERVER_RC_REPEAT_COUNT}" "${matrix_sequence}" "tuned"
|
|
cp "${tuned_result}" "${mode_result}"
|
|
run_status=${tuned_status}
|
|
else
|
|
echo " ↪ tune skipped: ${tune_reason:-not ready}"
|
|
fi
|
|
fi
|
|
|
|
if [[ "${run_status}" -ne 0 && "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then
|
|
break 2
|
|
fi
|
|
done
|
|
done
|
|
|
|
summarize_matrix
|
|
|
|
echo "==> done"
|
|
echo "artifact_dir: ${MATRIX_REPORT_DIR}"
|
|
echo "mode_matrix_summary_json: ${MATRIX_SUMMARY_JSON}"
|
|
echo "mode_matrix_summary_csv: ${MATRIX_SUMMARY_CSV}"
|
|
echo "mode_matrix_summary_txt: ${MATRIX_SUMMARY_TXT}"
|
|
echo "mode_delay_recommendations_json: ${MATRIX_DELAY_JSON}"
|
|
echo "mode_delay_recommendations_env: ${MATRIX_DELAY_ENV}"
|
|
echo "mode_static_calibration_json: ${MATRIX_STATIC_JSON}"
|
|
echo "mode_static_calibration_csv: ${MATRIX_STATIC_CSV}"
|
|
echo "mode_static_calibration_txt: ${MATRIX_STATIC_TXT}"
|
|
echo "mode_static_calibration_env: ${MATRIX_STATIC_ENV}"
|
|
echo "mode_matrix_run_log: ${MATRIX_RUN_LOG}"
|
|
|
|
if python3 - <<'PY' "${MATRIX_SUMMARY_JSON}"
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
summary = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
raise SystemExit(0 if summary.get("passed") else 1)
|
|
PY
|
|
then
|
|
exit 0
|
|
fi
|
|
|
|
exit 95
|