617 lines
20 KiB
Bash
Executable File
617 lines
20 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# scripts/manual/run_upstream_av_sync.sh
|
|
#
|
|
# Manual: capture the real Tethys webcam/mic endpoints while the shared-clock
|
|
# sync probe streams upstream media through Lesavka, then analyze the skew.
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
|
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"
|
|
|
|
TETHYS_HOST=${TETHYS_HOST:-tethys}
|
|
LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST:-theia}
|
|
LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-http://38.28.125.112:50051}
|
|
PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-10}
|
|
PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4}
|
|
LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-8}
|
|
TAIL_SECONDS=${TAIL_SECONDS:-2}
|
|
CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))}
|
|
REMOTE_CAPTURE=${REMOTE_CAPTURE:-/tmp/lesavka-upstream-av-sync.mkv}
|
|
LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-"${REPO_ROOT}/tmp"}
|
|
VIDEO_SIZE=${VIDEO_SIZE:-auto}
|
|
VIDEO_FPS=${VIDEO_FPS:-30}
|
|
VIDEO_FORMAT=${VIDEO_FORMAT:-mjpeg}
|
|
REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse}
|
|
REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-ffmpeg}
|
|
REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-copy}
|
|
REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto}
|
|
REMOTE_AUDIO_QUIESCE_USER_AUDIO=${REMOTE_AUDIO_QUIESCE_USER_AUDIO:-auto}
|
|
ANALYSIS_NORMALIZE=${ANALYSIS_NORMALIZE:-0}
|
|
ANALYSIS_SCALE_WIDTH=${ANALYSIS_SCALE_WIDTH:-1280}
|
|
SSH_OPTS=${SSH_OPTS:-"-o BatchMode=yes -o ConnectTimeout=30"}
|
|
LOCAL_AUDIO_SANITY=${LOCAL_AUDIO_SANITY:-1}
|
|
PROBE_PREBUILD=${PROBE_PREBUILD:-1}
|
|
PROBE_BIN=${PROBE_BIN:-"${REPO_ROOT}/target/debug/lesavka-sync-probe"}
|
|
ANALYZE_BIN=${ANALYZE_BIN:-"${REPO_ROOT}/target/debug/lesavka-sync-analyze"}
|
|
REMOTE_ANALYZE=${REMOTE_ANALYZE:-1}
|
|
REMOTE_ANALYZE_BIN=${REMOTE_ANALYZE_BIN:-/tmp/lesavka-sync-analyze}
|
|
REMOTE_ANALYZE_COPY=${REMOTE_ANALYZE_COPY:-1}
|
|
FETCH_CAPTURE=${FETCH_CAPTURE:-0}
|
|
REMOTE_SERVER_PREFLIGHT=${REMOTE_SERVER_PREFLIGHT:-1}
|
|
REMOTE_EXPECT_CAM_OUTPUT=${REMOTE_EXPECT_CAM_OUTPUT:-uvc}
|
|
REMOTE_EXPECT_UVC_CODEC=${REMOTE_EXPECT_UVC_CODEC:-mjpeg}
|
|
|
|
mkdir -p "${LOCAL_OUTPUT_DIR}"
|
|
STAMP="$(date +%Y%m%d-%H%M%S)"
|
|
LOCAL_CAPTURE="${LOCAL_OUTPUT_DIR}/lesavka-upstream-av-sync-${STAMP}.mkv"
|
|
LOCAL_ANALYSIS_JSON="${LOCAL_CAPTURE%.mkv}.json"
|
|
LOCAL_CAPTURE_LOG="${LOCAL_CAPTURE%.mkv}.capture.log"
|
|
|
|
preflight_server_path() {
|
|
[[ "${REMOTE_SERVER_PREFLIGHT}" != "0" ]] || return 0
|
|
|
|
echo "==> verifying Lesavka server path on ${LESAVKA_SERVER_HOST}"
|
|
ssh ${SSH_OPTS} "${LESAVKA_SERVER_HOST}" bash -s -- \
|
|
"${REMOTE_EXPECT_CAM_OUTPUT}" \
|
|
"${REMOTE_EXPECT_UVC_CODEC}" <<'REMOTE_PREFLIGHT'
|
|
set -euo pipefail
|
|
expect_cam_output=$1
|
|
expect_uvc_codec=$2
|
|
|
|
read_env_value() {
|
|
local key=$1
|
|
local file=$2
|
|
local value=""
|
|
value=$(grep -E "^${key}=" "$file" 2>/dev/null | tail -n1 | cut -d= -f2- || true)
|
|
printf '%s\n' "$value"
|
|
}
|
|
|
|
cam_output=$(read_env_value "LESAVKA_CAM_OUTPUT" /etc/lesavka/server.env)
|
|
server_uvc_codec=$(read_env_value "LESAVKA_UVC_CODEC" /etc/lesavka/server.env)
|
|
runtime_uvc_codec=$(read_env_value "LESAVKA_UVC_CODEC" /etc/lesavka/uvc.env)
|
|
|
|
printf ' ↪ server.env CAM_OUTPUT=%s\n' "${cam_output:-<unset>}"
|
|
printf ' ↪ server.env UVC_CODEC=%s\n' "${server_uvc_codec:-<unset>}"
|
|
printf ' ↪ uvc.env UVC_CODEC=%s\n' "${runtime_uvc_codec:-<unset>}"
|
|
|
|
if [[ -n "${expect_cam_output}" && "${cam_output}" != "${expect_cam_output}" ]]; then
|
|
printf 'expected CAM_OUTPUT=%s but found %s\n' "${expect_cam_output}" "${cam_output:-<unset>}" >&2
|
|
exit 64
|
|
fi
|
|
if [[ -n "${expect_uvc_codec}" && "${server_uvc_codec}" != "${expect_uvc_codec}" ]]; then
|
|
printf 'expected server.env UVC_CODEC=%s but found %s\n' "${expect_uvc_codec}" "${server_uvc_codec:-<unset>}" >&2
|
|
exit 65
|
|
fi
|
|
if [[ -n "${expect_uvc_codec}" && "${runtime_uvc_codec}" != "${expect_uvc_codec}" ]]; then
|
|
printf 'expected uvc.env UVC_CODEC=%s but found %s\n' "${expect_uvc_codec}" "${runtime_uvc_codec:-<unset>}" >&2
|
|
exit 66
|
|
fi
|
|
|
|
systemctl is-active lesavka-server lesavka-uvc lesavka-core >/dev/null
|
|
REMOTE_PREFLIGHT
|
|
}
|
|
|
|
if [[ "${LOCAL_AUDIO_SANITY}" != "0" ]]; then
|
|
echo "==> verifying local speaker-to-mic sanity before upstream sync run"
|
|
"${SCRIPT_DIR}/run_local_audio_sanity.sh"
|
|
fi
|
|
|
|
if [[ "${PROBE_PREBUILD}" != "0" ]]; then
|
|
echo "==> prebuilding sync probe/analyzer before opening the capture window"
|
|
(
|
|
cd "${REPO_ROOT}"
|
|
cargo build -p lesavka_client --bin lesavka-sync-probe --bin lesavka-sync-analyze
|
|
)
|
|
fi
|
|
|
|
if [[ ! -x "${PROBE_BIN}" ]]; then
|
|
echo "sync probe binary not found at ${PROBE_BIN}" >&2
|
|
exit 1
|
|
fi
|
|
if [[ ! -x "${ANALYZE_BIN}" ]]; then
|
|
echo "sync analyzer binary not found at ${ANALYZE_BIN}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
preflight_server_path
|
|
|
|
echo "==> starting Tethys capture on ${TETHYS_HOST}"
|
|
ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
|
|
"${REMOTE_CAPTURE}" \
|
|
"${CAPTURE_SECONDS}" \
|
|
"${VIDEO_SIZE}" \
|
|
"${VIDEO_FPS}" \
|
|
"${VIDEO_FORMAT}" \
|
|
"${REMOTE_CAPTURE_STACK}" \
|
|
"${REMOTE_PULSE_CAPTURE_TOOL}" \
|
|
"${REMOTE_PULSE_VIDEO_MODE}" \
|
|
"${REMOTE_AUDIO_SOURCE}" \
|
|
"${REMOTE_AUDIO_QUIESCE_USER_AUDIO}" \
|
|
> >(tee "${LOCAL_CAPTURE_LOG}") \
|
|
2> >(tee -a "${LOCAL_CAPTURE_LOG}" >&2) <<'REMOTE_CAPTURE_SCRIPT' &
|
|
set -euo pipefail
|
|
remote_capture=$1
|
|
capture_seconds=$2
|
|
video_size=$3
|
|
video_fps=$4
|
|
video_format=$5
|
|
remote_capture_stack=$6
|
|
remote_pulse_capture_tool=$7
|
|
remote_pulse_video_mode=$8
|
|
remote_audio_source=$9
|
|
remote_audio_quiesce_user_audio=${10}
|
|
|
|
rm -f "${remote_capture}"
|
|
|
|
restore_user_audio() {
|
|
systemctl --user start pipewire.socket pipewire-pulse.socket wireplumber.service >/dev/null 2>&1 || true
|
|
sleep 1
|
|
systemctl --user start pipewire.service pipewire-pulse.service >/dev/null 2>&1 || true
|
|
}
|
|
|
|
quiesce_user_audio() {
|
|
systemctl --user stop pipewire-pulse.service pipewire.service wireplumber.service \
|
|
pipewire-pulse.socket pipewire.socket >/dev/null 2>&1 || true
|
|
sleep 1
|
|
}
|
|
|
|
resolve_pulse_source() {
|
|
if ! command -v pactl >/dev/null 2>&1; then
|
|
return 1
|
|
fi
|
|
pactl list short sources 2>/dev/null \
|
|
| awk '
|
|
/alsa_input\..*Lesavka_Composite/ { print $2; found=1; exit }
|
|
/Lesavka_Composite/ && !fallback { fallback=$2 }
|
|
END {
|
|
if (found) exit 0
|
|
if (fallback != "") { print fallback; exit 0 }
|
|
exit 1
|
|
}
|
|
'
|
|
}
|
|
|
|
gst_video_source_caps() {
|
|
case "${video_format}" in
|
|
""|mjpeg|MJPG)
|
|
printf 'image/jpeg,width=%s,height=%s,framerate=%s/1' \
|
|
"${resolved_video_size%x*}" \
|
|
"${resolved_video_size#*x}" \
|
|
"${video_fps}"
|
|
;;
|
|
yuyv422|YUYV|yuyv)
|
|
printf 'video/x-raw,format=YUY2,width=%s,height=%s,framerate=%s/1' \
|
|
"${resolved_video_size%x*}" \
|
|
"${resolved_video_size#*x}" \
|
|
"${video_fps}"
|
|
;;
|
|
*)
|
|
printf 'unsupported gst video_format=%s\n' "${video_format}" >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
}
|
|
|
|
gst_video_decode_chain() {
|
|
case "${video_format}" in
|
|
""|mjpeg|MJPG)
|
|
printf 'jpegdec ! '
|
|
;;
|
|
yuyv422|YUYV|yuyv)
|
|
printf ''
|
|
;;
|
|
*)
|
|
printf 'unsupported gst video_format=%s\n' "${video_format}" >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
}
|
|
|
|
resolve_video_size() {
|
|
local requested=$1
|
|
if [[ "${requested}" != "auto" ]]; then
|
|
printf '%s\n' "${requested}"
|
|
return 0
|
|
fi
|
|
if ! command -v v4l2-ctl >/dev/null 2>&1; then
|
|
printf '1280x720\n'
|
|
return 0
|
|
fi
|
|
local listing
|
|
listing="$(v4l2-ctl -d /dev/video0 --list-formats-ext 2>/dev/null || true)"
|
|
local preferred
|
|
for preferred in 1920x1080 1360x768 1280x720; do
|
|
if grep -q "Size: Discrete ${preferred}" <<<"${listing}"; then
|
|
printf '%s\n' "${preferred}"
|
|
return 0
|
|
fi
|
|
done
|
|
printf '1280x720\n'
|
|
}
|
|
|
|
resolve_pw_audio_target() {
|
|
if ! command -v pw-dump >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then
|
|
return 1
|
|
fi
|
|
pw-dump | python3 -c '
|
|
import json
|
|
import sys
|
|
|
|
try:
|
|
objs = json.load(sys.stdin)
|
|
except Exception:
|
|
raise SystemExit(1)
|
|
|
|
for obj in objs:
|
|
if obj.get("type") != "PipeWire:Interface:Node":
|
|
continue
|
|
props = (obj.get("info") or {}).get("props") or {}
|
|
if props.get("media.class") != "Audio/Source":
|
|
continue
|
|
serial = props.get("object.serial")
|
|
name = props.get("node.name", "")
|
|
desc = props.get("node.description", "")
|
|
if serial is None:
|
|
continue
|
|
if "Lesavka_Composite" in name or "Lesavka Composite" in desc:
|
|
print(serial)
|
|
raise SystemExit(0)
|
|
|
|
raise SystemExit(1)
|
|
'
|
|
}
|
|
|
|
capture_mode="alsa"
|
|
alsa_audio_dev="hw:3,0"
|
|
pulse_source=""
|
|
pw_audio_target=""
|
|
|
|
case "${remote_capture_stack}" in
|
|
auto)
|
|
if [[ "${remote_audio_source}" == "auto" ]]; then
|
|
if pulse_source="$(resolve_pulse_source)"; then
|
|
capture_mode="pulse"
|
|
elif command -v pw-record >/dev/null 2>&1 \
|
|
&& command -v pw-v4l2 >/dev/null 2>&1 \
|
|
&& pw_audio_target="$(resolve_pw_audio_target)"; then
|
|
capture_mode="pwpipe"
|
|
else
|
|
printf 'PipeWire Lesavka source not found; falling back to hw:3,0\n' >&2
|
|
fi
|
|
elif [[ "${remote_audio_source}" == pulse:* ]]; then
|
|
capture_mode="pulse"
|
|
pulse_source="${remote_audio_source#pulse:}"
|
|
elif [[ "${remote_audio_source}" == alsa:* ]]; then
|
|
alsa_audio_dev="${remote_audio_source#alsa:}"
|
|
else
|
|
printf 'unsupported REMOTE_AUDIO_SOURCE=%s\n' "${remote_audio_source}" >&2
|
|
exit 64
|
|
fi
|
|
;;
|
|
pwpipe)
|
|
if ! command -v pw-record >/dev/null 2>&1 || ! command -v pw-v4l2 >/dev/null 2>&1; then
|
|
printf 'REMOTE_CAPTURE_STACK=pwpipe requires pw-record and pw-v4l2\n' >&2
|
|
exit 64
|
|
fi
|
|
pw_audio_target="$(resolve_pw_audio_target)" || {
|
|
printf 'PipeWire Lesavka capture target not found for REMOTE_CAPTURE_STACK=pwpipe\n' >&2
|
|
exit 64
|
|
}
|
|
capture_mode="pwpipe"
|
|
;;
|
|
pulse)
|
|
if [[ "${remote_audio_source}" == pulse:* ]]; then
|
|
pulse_source="${remote_audio_source#pulse:}"
|
|
elif [[ "${remote_audio_source}" == "auto" ]]; then
|
|
pulse_source="$(resolve_pulse_source)" || {
|
|
printf 'PipeWire Lesavka source not found for REMOTE_CAPTURE_STACK=pulse\n' >&2
|
|
exit 64
|
|
}
|
|
else
|
|
pulse_source="${remote_audio_source}"
|
|
fi
|
|
capture_mode="pulse"
|
|
;;
|
|
alsa)
|
|
if [[ "${remote_audio_source}" == alsa:* ]]; then
|
|
alsa_audio_dev="${remote_audio_source#alsa:}"
|
|
elif [[ "${remote_audio_source}" != "auto" ]]; then
|
|
alsa_audio_dev="${remote_audio_source}"
|
|
fi
|
|
capture_mode="alsa"
|
|
;;
|
|
*)
|
|
printf 'unsupported REMOTE_CAPTURE_STACK=%s\n' "${remote_capture_stack}" >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
|
|
resolved_video_size="$(resolve_video_size "${video_size}")"
|
|
printf 'using video mode: %s (%s)\n' "${resolved_video_size}" "${video_format:-driver-default}" >&2
|
|
video_args=(-f video4linux2 -framerate "${video_fps}" -video_size "${resolved_video_size}")
|
|
if [[ -n "${video_format}" ]]; then
|
|
video_args+=(-input_format "${video_format}")
|
|
fi
|
|
gst_source_caps="$(gst_video_source_caps)"
|
|
gst_decode_chain="$(gst_video_decode_chain)"
|
|
|
|
quiesce_for_alsa=0
|
|
case "${remote_audio_quiesce_user_audio}" in
|
|
1|true|yes)
|
|
quiesce_for_alsa=1
|
|
;;
|
|
auto)
|
|
if [[ "${capture_mode}" == "alsa" ]]; then
|
|
quiesce_for_alsa=1
|
|
fi
|
|
;;
|
|
0|false|no)
|
|
quiesce_for_alsa=0
|
|
;;
|
|
*)
|
|
printf 'unsupported REMOTE_AUDIO_QUIESCE_USER_AUDIO=%s\n' "${remote_audio_quiesce_user_audio}" >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
|
|
if [[ "${capture_mode}" == "alsa" && "${quiesce_for_alsa}" == "1" ]]; then
|
|
printf 'quiescing Tethys user audio before raw ALSA capture\n' >&2
|
|
quiesce_user_audio
|
|
trap restore_user_audio EXIT
|
|
fi
|
|
|
|
if [[ "${capture_mode}" == "pwpipe" ]]; then
|
|
printf 'using PipeWire-native mux capture target serial: %s\n' "${pw_audio_target}" >&2
|
|
timeout "${capture_seconds}" pw-record \
|
|
--target "${pw_audio_target}" \
|
|
--rate 48000 \
|
|
--channels 2 \
|
|
--format s16 \
|
|
--raw - \
|
|
| pw-v4l2 ffmpeg -hide_banner -loglevel error -y \
|
|
-thread_queue_size 1024 \
|
|
"${video_args[@]}" \
|
|
-i /dev/video0 \
|
|
-thread_queue_size 1024 \
|
|
-f s16le -ar 48000 -ac 2 \
|
|
-i pipe:0 \
|
|
-t "${capture_seconds}" \
|
|
-c:v copy \
|
|
-c:a pcm_s16le \
|
|
"${remote_capture}"
|
|
elif [[ "${capture_mode}" == "pulse" ]]; then
|
|
printf 'using Pulse source: %s\n' "${pulse_source}" >&2
|
|
case "${remote_pulse_capture_tool}" in
|
|
ffmpeg)
|
|
case "${remote_pulse_video_mode}" in
|
|
copy)
|
|
ffmpeg -hide_banner -loglevel error -y \
|
|
-thread_queue_size 1024 \
|
|
"${video_args[@]}" \
|
|
-i /dev/video0 \
|
|
-thread_queue_size 1024 \
|
|
-f pulse \
|
|
-i "${pulse_source}" \
|
|
-t "${capture_seconds}" \
|
|
-c:v copy \
|
|
-c:a pcm_s16le \
|
|
"${remote_capture}"
|
|
;;
|
|
cfr)
|
|
ffmpeg -hide_banner -loglevel error -y \
|
|
-thread_queue_size 1024 \
|
|
"${video_args[@]}" \
|
|
-i /dev/video0 \
|
|
-thread_queue_size 1024 \
|
|
-f pulse \
|
|
-i "${pulse_source}" \
|
|
-t "${capture_seconds}" \
|
|
-vf "fps=${video_fps}" \
|
|
-c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \
|
|
-c:a pcm_s16le \
|
|
"${remote_capture}"
|
|
;;
|
|
*)
|
|
printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
;;
|
|
gst)
|
|
case "${remote_pulse_video_mode}" in
|
|
copy)
|
|
if [[ "${video_format}" != "mjpeg" && "${video_format}" != "MJPG" && -n "${video_format}" ]]; then
|
|
printf 'gst copy mode only supports mjpeg input, got %s\n' "${video_format}" >&2
|
|
exit 64
|
|
fi
|
|
timeout --signal=INT "$((capture_seconds + 3))" \
|
|
gst-launch-1.0 -q -e \
|
|
matroskamux name=mux ! filesink location="${remote_capture}" \
|
|
v4l2src device=/dev/video0 do-timestamp=true ! \
|
|
${gst_source_caps} ! \
|
|
queue ! mux. \
|
|
pulsesrc device="${pulse_source}" do-timestamp=true ! \
|
|
audio/x-raw,rate=48000,channels=2 ! \
|
|
audioconvert ! audioresample ! queue ! mux. || true
|
|
;;
|
|
cfr)
|
|
timeout --signal=INT "$((capture_seconds + 3))" \
|
|
gst-launch-1.0 -q -e \
|
|
matroskamux name=mux ! filesink location="${remote_capture}" \
|
|
v4l2src device=/dev/video0 do-timestamp=true ! \
|
|
${gst_source_caps} ! \
|
|
${gst_decode_chain} \
|
|
videoconvert ! videorate ! video/x-raw,framerate="${video_fps}"/1 ! \
|
|
x264enc tune=zerolatency speed-preset=ultrafast key-int-max=1 bitrate=5000 ! \
|
|
h264parse ! \
|
|
queue ! mux. \
|
|
pulsesrc device="${pulse_source}" do-timestamp=true ! \
|
|
audio/x-raw,rate=48000,channels=2 ! \
|
|
audioconvert ! audioresample ! queue ! mux. || true
|
|
;;
|
|
*)
|
|
printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
;;
|
|
*)
|
|
printf 'unsupported REMOTE_PULSE_CAPTURE_TOOL=%s\n' "${remote_pulse_capture_tool}" >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
else
|
|
ffmpeg -hide_banner -loglevel error -y \
|
|
-thread_queue_size 1024 \
|
|
"${video_args[@]}" \
|
|
-i /dev/video0 \
|
|
-thread_queue_size 1024 \
|
|
-f alsa -ac 2 -ar 48000 \
|
|
-i "${alsa_audio_dev}" \
|
|
-t "${capture_seconds}" \
|
|
-c:v ffv1 -level 3 -g 1 \
|
|
-c:a pcm_s16le \
|
|
"${remote_capture}"
|
|
fi
|
|
REMOTE_CAPTURE_SCRIPT
|
|
capture_pid=$!
|
|
|
|
sleep "${LEAD_IN_SECONDS}"
|
|
|
|
echo "==> running local Lesavka sync probe against ${LESAVKA_SERVER_ADDR}"
|
|
probe_status=0
|
|
(
|
|
cd "${REPO_ROOT}"
|
|
"${PROBE_BIN}" \
|
|
--server "${LESAVKA_SERVER_ADDR}" \
|
|
--duration-seconds "${PROBE_DURATION_SECONDS}" \
|
|
--warmup-seconds "${PROBE_WARMUP_SECONDS}"
|
|
) || probe_status=$?
|
|
|
|
capture_status=0
|
|
wait "${capture_pid}" || capture_status=$?
|
|
capture_v4l2_fault=0
|
|
if [[ -f "${LOCAL_CAPTURE_LOG}" ]] \
|
|
&& grep -q 'VIDIOC_QBUF): Bad file descriptor' "${LOCAL_CAPTURE_LOG}"; then
|
|
capture_v4l2_fault=1
|
|
fi
|
|
|
|
if ssh ${SSH_OPTS} "${TETHYS_HOST}" "test -f '${REMOTE_CAPTURE}'"; then
|
|
remote_fetch_capture="${REMOTE_CAPTURE}"
|
|
if [[ "${ANALYSIS_NORMALIZE}" != "0" ]]; then
|
|
remote_fetch_capture="${REMOTE_CAPTURE%.mkv}-analysis.mkv"
|
|
echo "==> normalizing remote capture to CFR for analysis"
|
|
normalize_status=0
|
|
ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
|
|
"${REMOTE_CAPTURE}" \
|
|
"${remote_fetch_capture}" \
|
|
"${VIDEO_FPS}" \
|
|
"${ANALYSIS_SCALE_WIDTH}" <<'REMOTE_NORMALIZE_SCRIPT' || normalize_status=$?
|
|
set -euo pipefail
|
|
src=$1
|
|
dst=$2
|
|
fps=$3
|
|
scale_width=$4
|
|
ffmpeg -hide_banner -loglevel error -y \
|
|
-i "${src}" \
|
|
-vf "fps=${fps},scale='min(${scale_width},iw)':-2" \
|
|
-c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \
|
|
-c:a pcm_s16le \
|
|
"${dst}"
|
|
REMOTE_NORMALIZE_SCRIPT
|
|
if [[ "${normalize_status}" -ne 0 ]]; then
|
|
echo "remote CFR normalization failed; falling back to raw capture" >&2
|
|
remote_fetch_capture="${REMOTE_CAPTURE}"
|
|
fi
|
|
fi
|
|
|
|
if [[ "${REMOTE_ANALYZE}" != "0" ]]; then
|
|
if [[ "${REMOTE_ANALYZE_COPY}" != "0" ]]; then
|
|
echo "==> copying sync analyzer to ${TETHYS_HOST}:${REMOTE_ANALYZE_BIN}"
|
|
scp ${SSH_OPTS} "${ANALYZE_BIN}" "${TETHYS_HOST}:${REMOTE_ANALYZE_BIN}"
|
|
fi
|
|
echo "==> analyzing capture on ${TETHYS_HOST}"
|
|
ssh ${SSH_OPTS} "${TETHYS_HOST}" \
|
|
"chmod +x '${REMOTE_ANALYZE_BIN}' && '${REMOTE_ANALYZE_BIN}' '${remote_fetch_capture}' --json" \
|
|
> "${LOCAL_ANALYSIS_JSON}"
|
|
fi
|
|
|
|
if [[ "${FETCH_CAPTURE}" != "0" ]]; then
|
|
echo "==> fetching capture back to ${LOCAL_CAPTURE}"
|
|
scp ${SSH_OPTS} "${TETHYS_HOST}:${remote_fetch_capture}" "${LOCAL_CAPTURE}"
|
|
fi
|
|
fi
|
|
|
|
if [[ "${probe_status}" -ne 0 ]]; then
|
|
echo "sync probe failed with status ${probe_status}" >&2
|
|
[[ -f "${LOCAL_CAPTURE}" ]] && echo "partial capture preserved at ${LOCAL_CAPTURE}" >&2
|
|
exit "${probe_status}"
|
|
fi
|
|
if [[ "${capture_status}" -ne 0 ]]; then
|
|
if [[ "${capture_status}" -eq 141 && ( -f "${LOCAL_CAPTURE}" || -f "${LOCAL_ANALYSIS_JSON}" ) ]]; then
|
|
echo "Tethys capture ended with PipeWire SIGPIPE after ffmpeg closed; accepting preserved analysis artifacts" >&2
|
|
elif [[ "${capture_status}" -eq 124 && ( -f "${LOCAL_CAPTURE}" || -f "${LOCAL_ANALYSIS_JSON}" ) ]]; then
|
|
echo "Tethys capture timed out after preserving analysis artifacts; accepting the run for analysis" >&2
|
|
else
|
|
echo "Tethys capture failed with status ${capture_status}" >&2
|
|
[[ -f "${LOCAL_CAPTURE}" ]] && echo "partial capture preserved at ${LOCAL_CAPTURE}" >&2
|
|
exit "${capture_status}"
|
|
fi
|
|
fi
|
|
|
|
if [[ "${REMOTE_ANALYZE}" != "0" ]]; then
|
|
if [[ ! -f "${LOCAL_ANALYSIS_JSON}" ]]; then
|
|
echo "remote analysis did not produce ${LOCAL_ANALYSIS_JSON}" >&2
|
|
exit 92
|
|
fi
|
|
echo "==> remote analysis summary"
|
|
python - <<'PY' "${LOCAL_ANALYSIS_JSON}"
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
report = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
print(f"A/V sync report for {sys.argv[1]}")
|
|
print(f"- video onsets: {report['video_event_count']}")
|
|
print(f"- audio onsets: {report['audio_event_count']}")
|
|
print(f"- paired pulses: {report['paired_event_count']}")
|
|
print(f"- first skew: {report['first_skew_ms']:+.1f} ms (audio after video is positive)")
|
|
print(f"- last skew: {report['last_skew_ms']:+.1f} ms")
|
|
print(f"- mean skew: {report['mean_skew_ms']:+.1f} ms")
|
|
print(f"- median skew: {report['median_skew_ms']:+.1f} ms")
|
|
print(f"- max abs skew: {report['max_abs_skew_ms']:.1f} ms")
|
|
print(f"- drift: {report['drift_ms']:+.1f} ms")
|
|
cal = report.get('calibration', {})
|
|
print(f"- calibration ready: {cal.get('ready')}")
|
|
print(f"- recommended audio offset adjust: {int(cal.get('recommended_audio_offset_adjust_us', 0)):+d} us")
|
|
print(f"- alternative video offset adjust: {int(cal.get('recommended_video_offset_adjust_us', 0)):+d} us")
|
|
print(f"- calibration note: {cal.get('note', '')}")
|
|
PY
|
|
else
|
|
if [[ ! -f "${LOCAL_CAPTURE}" ]]; then
|
|
echo "capture was not fetched and REMOTE_ANALYZE=0 left nothing local to analyze" >&2
|
|
exit 93
|
|
fi
|
|
echo "==> analyzing capture"
|
|
(
|
|
cd "${REPO_ROOT}"
|
|
"${ANALYZE_BIN}" "${LOCAL_CAPTURE}"
|
|
)
|
|
fi
|
|
|
|
if [[ "${capture_v4l2_fault}" -eq 1 ]]; then
|
|
echo "warning: Tethys video capture reported VIDIOC_QBUF / Bad file descriptor; treat unstable skew or analyzer failures as host-capture suspect" >&2
|
|
fi
|
|
|
|
echo "==> done"
|
|
if [[ -f "${LOCAL_CAPTURE}" ]]; then
|
|
echo "capture: ${LOCAL_CAPTURE}"
|
|
fi
|
|
if [[ -f "${LOCAL_ANALYSIS_JSON}" ]]; then
|
|
echo "analysis_json: ${LOCAL_ANALYSIS_JSON}"
|
|
fi
|
|
if [[ -f "${LOCAL_CAPTURE_LOG}" ]]; then
|
|
echo "capture_log: ${LOCAL_CAPTURE_LOG}"
|
|
fi
|