lesavka/scripts/manual/run_upstream_av_sync.sh

567 lines
18 KiB
Bash
Executable File

#!/usr/bin/env bash
# scripts/manual/run_upstream_av_sync.sh
#
# Manual: capture the real Tethys webcam/mic endpoints while the shared-clock
# sync probe streams upstream media through Lesavka, then analyze the skew.
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"
TETHYS_HOST=${TETHYS_HOST:-tethys}
LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-http://38.28.125.112:50051}
PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-10}
PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4}
LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-8}
TAIL_SECONDS=${TAIL_SECONDS:-2}
CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))}
REMOTE_CAPTURE=${REMOTE_CAPTURE:-/tmp/lesavka-upstream-av-sync.mkv}
LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-"${REPO_ROOT}/tmp"}
VIDEO_SIZE=${VIDEO_SIZE:-auto}
VIDEO_FPS=${VIDEO_FPS:-30}
VIDEO_FORMAT=${VIDEO_FORMAT:-mjpeg}
REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse}
REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-ffmpeg}
REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-copy}
REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto}
REMOTE_AUDIO_QUIESCE_USER_AUDIO=${REMOTE_AUDIO_QUIESCE_USER_AUDIO:-auto}
ANALYSIS_NORMALIZE=${ANALYSIS_NORMALIZE:-0}
ANALYSIS_SCALE_WIDTH=${ANALYSIS_SCALE_WIDTH:-1280}
SSH_OPTS=${SSH_OPTS:-"-o BatchMode=yes -o ConnectTimeout=30"}
LOCAL_AUDIO_SANITY=${LOCAL_AUDIO_SANITY:-1}
PROBE_PREBUILD=${PROBE_PREBUILD:-1}
PROBE_BIN=${PROBE_BIN:-"${REPO_ROOT}/target/debug/lesavka-sync-probe"}
ANALYZE_BIN=${ANALYZE_BIN:-"${REPO_ROOT}/target/debug/lesavka-sync-analyze"}
REMOTE_ANALYZE=${REMOTE_ANALYZE:-1}
REMOTE_ANALYZE_BIN=${REMOTE_ANALYZE_BIN:-/tmp/lesavka-sync-analyze}
REMOTE_ANALYZE_COPY=${REMOTE_ANALYZE_COPY:-1}
FETCH_CAPTURE=${FETCH_CAPTURE:-0}
mkdir -p "${LOCAL_OUTPUT_DIR}"
STAMP="$(date +%Y%m%d-%H%M%S)"
LOCAL_CAPTURE="${LOCAL_OUTPUT_DIR}/lesavka-upstream-av-sync-${STAMP}.mkv"
LOCAL_ANALYSIS_JSON="${LOCAL_CAPTURE%.mkv}.json"
LOCAL_CAPTURE_LOG="${LOCAL_CAPTURE%.mkv}.capture.log"
if [[ "${LOCAL_AUDIO_SANITY}" != "0" ]]; then
echo "==> verifying local speaker-to-mic sanity before upstream sync run"
"${SCRIPT_DIR}/run_local_audio_sanity.sh"
fi
if [[ "${PROBE_PREBUILD}" != "0" ]]; then
echo "==> prebuilding sync probe/analyzer before opening the capture window"
(
cd "${REPO_ROOT}"
cargo build -p lesavka_client --bin lesavka-sync-probe --bin lesavka-sync-analyze
)
fi
if [[ ! -x "${PROBE_BIN}" ]]; then
echo "sync probe binary not found at ${PROBE_BIN}" >&2
exit 1
fi
if [[ ! -x "${ANALYZE_BIN}" ]]; then
echo "sync analyzer binary not found at ${ANALYZE_BIN}" >&2
exit 1
fi
echo "==> starting Tethys capture on ${TETHYS_HOST}"
ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
"${REMOTE_CAPTURE}" \
"${CAPTURE_SECONDS}" \
"${VIDEO_SIZE}" \
"${VIDEO_FPS}" \
"${VIDEO_FORMAT}" \
"${REMOTE_CAPTURE_STACK}" \
"${REMOTE_PULSE_CAPTURE_TOOL}" \
"${REMOTE_PULSE_VIDEO_MODE}" \
"${REMOTE_AUDIO_SOURCE}" \
"${REMOTE_AUDIO_QUIESCE_USER_AUDIO}" \
> >(tee "${LOCAL_CAPTURE_LOG}") \
2> >(tee -a "${LOCAL_CAPTURE_LOG}" >&2) <<'REMOTE_CAPTURE_SCRIPT' &
set -euo pipefail
remote_capture=$1
capture_seconds=$2
video_size=$3
video_fps=$4
video_format=$5
remote_capture_stack=$6
remote_pulse_capture_tool=$7
remote_pulse_video_mode=$8
remote_audio_source=$9
remote_audio_quiesce_user_audio=${10}
rm -f "${remote_capture}"
restore_user_audio() {
systemctl --user start pipewire.socket pipewire-pulse.socket wireplumber.service >/dev/null 2>&1 || true
sleep 1
systemctl --user start pipewire.service pipewire-pulse.service >/dev/null 2>&1 || true
}
quiesce_user_audio() {
systemctl --user stop pipewire-pulse.service pipewire.service wireplumber.service \
pipewire-pulse.socket pipewire.socket >/dev/null 2>&1 || true
sleep 1
}
resolve_pulse_source() {
if ! command -v pactl >/dev/null 2>&1; then
return 1
fi
pactl list short sources 2>/dev/null \
| awk '
/alsa_input\..*Lesavka_Composite/ { print $2; found=1; exit }
/Lesavka_Composite/ && !fallback { fallback=$2 }
END {
if (found) exit 0
if (fallback != "") { print fallback; exit 0 }
exit 1
}
'
}
gst_video_source_caps() {
case "${video_format}" in
""|mjpeg|MJPG)
printf 'image/jpeg,width=%s,height=%s,framerate=%s/1' \
"${resolved_video_size%x*}" \
"${resolved_video_size#*x}" \
"${video_fps}"
;;
yuyv422|YUYV|yuyv)
printf 'video/x-raw,format=YUY2,width=%s,height=%s,framerate=%s/1' \
"${resolved_video_size%x*}" \
"${resolved_video_size#*x}" \
"${video_fps}"
;;
*)
printf 'unsupported gst video_format=%s\n' "${video_format}" >&2
exit 64
;;
esac
}
gst_video_decode_chain() {
case "${video_format}" in
""|mjpeg|MJPG)
printf 'jpegdec ! '
;;
yuyv422|YUYV|yuyv)
printf ''
;;
*)
printf 'unsupported gst video_format=%s\n' "${video_format}" >&2
exit 64
;;
esac
}
resolve_video_size() {
local requested=$1
if [[ "${requested}" != "auto" ]]; then
printf '%s\n' "${requested}"
return 0
fi
if ! command -v v4l2-ctl >/dev/null 2>&1; then
printf '1280x720\n'
return 0
fi
local listing
listing="$(v4l2-ctl -d /dev/video0 --list-formats-ext 2>/dev/null || true)"
local preferred
for preferred in 1920x1080 1360x768 1280x720; do
if grep -q "Size: Discrete ${preferred}" <<<"${listing}"; then
printf '%s\n' "${preferred}"
return 0
fi
done
printf '1280x720\n'
}
resolve_pw_audio_target() {
if ! command -v pw-dump >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then
return 1
fi
pw-dump | python3 -c '
import json
import sys
try:
objs = json.load(sys.stdin)
except Exception:
raise SystemExit(1)
for obj in objs:
if obj.get("type") != "PipeWire:Interface:Node":
continue
props = (obj.get("info") or {}).get("props") or {}
if props.get("media.class") != "Audio/Source":
continue
serial = props.get("object.serial")
name = props.get("node.name", "")
desc = props.get("node.description", "")
if serial is None:
continue
if "Lesavka_Composite" in name or "Lesavka Composite" in desc:
print(serial)
raise SystemExit(0)
raise SystemExit(1)
'
}
capture_mode="alsa"
alsa_audio_dev="hw:3,0"
pulse_source=""
pw_audio_target=""
case "${remote_capture_stack}" in
auto)
if [[ "${remote_audio_source}" == "auto" ]]; then
if pulse_source="$(resolve_pulse_source)"; then
capture_mode="pulse"
elif command -v pw-record >/dev/null 2>&1 \
&& command -v pw-v4l2 >/dev/null 2>&1 \
&& pw_audio_target="$(resolve_pw_audio_target)"; then
capture_mode="pwpipe"
else
printf 'PipeWire Lesavka source not found; falling back to hw:3,0\n' >&2
fi
elif [[ "${remote_audio_source}" == pulse:* ]]; then
capture_mode="pulse"
pulse_source="${remote_audio_source#pulse:}"
elif [[ "${remote_audio_source}" == alsa:* ]]; then
alsa_audio_dev="${remote_audio_source#alsa:}"
else
printf 'unsupported REMOTE_AUDIO_SOURCE=%s\n' "${remote_audio_source}" >&2
exit 64
fi
;;
pwpipe)
if ! command -v pw-record >/dev/null 2>&1 || ! command -v pw-v4l2 >/dev/null 2>&1; then
printf 'REMOTE_CAPTURE_STACK=pwpipe requires pw-record and pw-v4l2\n' >&2
exit 64
fi
pw_audio_target="$(resolve_pw_audio_target)" || {
printf 'PipeWire Lesavka capture target not found for REMOTE_CAPTURE_STACK=pwpipe\n' >&2
exit 64
}
capture_mode="pwpipe"
;;
pulse)
if [[ "${remote_audio_source}" == pulse:* ]]; then
pulse_source="${remote_audio_source#pulse:}"
elif [[ "${remote_audio_source}" == "auto" ]]; then
pulse_source="$(resolve_pulse_source)" || {
printf 'PipeWire Lesavka source not found for REMOTE_CAPTURE_STACK=pulse\n' >&2
exit 64
}
else
pulse_source="${remote_audio_source}"
fi
capture_mode="pulse"
;;
alsa)
if [[ "${remote_audio_source}" == alsa:* ]]; then
alsa_audio_dev="${remote_audio_source#alsa:}"
elif [[ "${remote_audio_source}" != "auto" ]]; then
alsa_audio_dev="${remote_audio_source}"
fi
capture_mode="alsa"
;;
*)
printf 'unsupported REMOTE_CAPTURE_STACK=%s\n' "${remote_capture_stack}" >&2
exit 64
;;
esac
resolved_video_size="$(resolve_video_size "${video_size}")"
printf 'using video mode: %s (%s)\n' "${resolved_video_size}" "${video_format:-driver-default}" >&2
video_args=(-f video4linux2 -framerate "${video_fps}" -video_size "${resolved_video_size}")
if [[ -n "${video_format}" ]]; then
video_args+=(-input_format "${video_format}")
fi
gst_source_caps="$(gst_video_source_caps)"
gst_decode_chain="$(gst_video_decode_chain)"
quiesce_for_alsa=0
case "${remote_audio_quiesce_user_audio}" in
1|true|yes)
quiesce_for_alsa=1
;;
auto)
if [[ "${capture_mode}" == "alsa" ]]; then
quiesce_for_alsa=1
fi
;;
0|false|no)
quiesce_for_alsa=0
;;
*)
printf 'unsupported REMOTE_AUDIO_QUIESCE_USER_AUDIO=%s\n' "${remote_audio_quiesce_user_audio}" >&2
exit 64
;;
esac
if [[ "${capture_mode}" == "alsa" && "${quiesce_for_alsa}" == "1" ]]; then
printf 'quiescing Tethys user audio before raw ALSA capture\n' >&2
quiesce_user_audio
trap restore_user_audio EXIT
fi
if [[ "${capture_mode}" == "pwpipe" ]]; then
printf 'using PipeWire-native mux capture target serial: %s\n' "${pw_audio_target}" >&2
timeout "${capture_seconds}" pw-record \
--target "${pw_audio_target}" \
--rate 48000 \
--channels 2 \
--format s16 \
--raw - \
| pw-v4l2 ffmpeg -hide_banner -loglevel error -y \
-thread_queue_size 1024 \
"${video_args[@]}" \
-i /dev/video0 \
-thread_queue_size 1024 \
-f s16le -ar 48000 -ac 2 \
-i pipe:0 \
-t "${capture_seconds}" \
-c:v copy \
-c:a pcm_s16le \
"${remote_capture}"
elif [[ "${capture_mode}" == "pulse" ]]; then
printf 'using Pulse source: %s\n' "${pulse_source}" >&2
case "${remote_pulse_capture_tool}" in
ffmpeg)
case "${remote_pulse_video_mode}" in
copy)
ffmpeg -hide_banner -loglevel error -y \
-thread_queue_size 1024 \
"${video_args[@]}" \
-i /dev/video0 \
-thread_queue_size 1024 \
-f pulse \
-i "${pulse_source}" \
-t "${capture_seconds}" \
-c:v copy \
-c:a pcm_s16le \
"${remote_capture}"
;;
cfr)
ffmpeg -hide_banner -loglevel error -y \
-thread_queue_size 1024 \
"${video_args[@]}" \
-i /dev/video0 \
-thread_queue_size 1024 \
-f pulse \
-i "${pulse_source}" \
-t "${capture_seconds}" \
-vf "fps=${video_fps}" \
-c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \
-c:a pcm_s16le \
"${remote_capture}"
;;
*)
printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2
exit 64
;;
esac
;;
gst)
case "${remote_pulse_video_mode}" in
copy)
if [[ "${video_format}" != "mjpeg" && "${video_format}" != "MJPG" && -n "${video_format}" ]]; then
printf 'gst copy mode only supports mjpeg input, got %s\n' "${video_format}" >&2
exit 64
fi
timeout --signal=INT "$((capture_seconds + 3))" \
gst-launch-1.0 -q -e \
matroskamux name=mux ! filesink location="${remote_capture}" \
v4l2src device=/dev/video0 do-timestamp=true ! \
${gst_source_caps} ! \
queue ! mux. \
pulsesrc device="${pulse_source}" do-timestamp=true ! \
audio/x-raw,rate=48000,channels=2 ! \
audioconvert ! audioresample ! queue ! mux. || true
;;
cfr)
timeout --signal=INT "$((capture_seconds + 3))" \
gst-launch-1.0 -q -e \
matroskamux name=mux ! filesink location="${remote_capture}" \
v4l2src device=/dev/video0 do-timestamp=true ! \
${gst_source_caps} ! \
${gst_decode_chain} \
videoconvert ! videorate ! video/x-raw,framerate="${video_fps}"/1 ! \
x264enc tune=zerolatency speed-preset=ultrafast key-int-max=1 bitrate=5000 ! \
h264parse ! \
queue ! mux. \
pulsesrc device="${pulse_source}" do-timestamp=true ! \
audio/x-raw,rate=48000,channels=2 ! \
audioconvert ! audioresample ! queue ! mux. || true
;;
*)
printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2
exit 64
;;
esac
;;
*)
printf 'unsupported REMOTE_PULSE_CAPTURE_TOOL=%s\n' "${remote_pulse_capture_tool}" >&2
exit 64
;;
esac
else
ffmpeg -hide_banner -loglevel error -y \
-thread_queue_size 1024 \
"${video_args[@]}" \
-i /dev/video0 \
-thread_queue_size 1024 \
-f alsa -ac 2 -ar 48000 \
-i "${alsa_audio_dev}" \
-t "${capture_seconds}" \
-c:v ffv1 -level 3 -g 1 \
-c:a pcm_s16le \
"${remote_capture}"
fi
REMOTE_CAPTURE_SCRIPT
capture_pid=$!
sleep "${LEAD_IN_SECONDS}"
echo "==> running local Lesavka sync probe against ${LESAVKA_SERVER_ADDR}"
probe_status=0
(
cd "${REPO_ROOT}"
"${PROBE_BIN}" \
--server "${LESAVKA_SERVER_ADDR}" \
--duration-seconds "${PROBE_DURATION_SECONDS}" \
--warmup-seconds "${PROBE_WARMUP_SECONDS}"
) || probe_status=$?
capture_status=0
wait "${capture_pid}" || capture_status=$?
capture_v4l2_fault=0
if [[ -f "${LOCAL_CAPTURE_LOG}" ]] \
&& grep -q 'VIDIOC_QBUF): Bad file descriptor' "${LOCAL_CAPTURE_LOG}"; then
capture_v4l2_fault=1
fi
if ssh ${SSH_OPTS} "${TETHYS_HOST}" "test -f '${REMOTE_CAPTURE}'"; then
remote_fetch_capture="${REMOTE_CAPTURE}"
if [[ "${ANALYSIS_NORMALIZE}" != "0" ]]; then
remote_fetch_capture="${REMOTE_CAPTURE%.mkv}-analysis.mkv"
echo "==> normalizing remote capture to CFR for analysis"
normalize_status=0
ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
"${REMOTE_CAPTURE}" \
"${remote_fetch_capture}" \
"${VIDEO_FPS}" \
"${ANALYSIS_SCALE_WIDTH}" <<'REMOTE_NORMALIZE_SCRIPT' || normalize_status=$?
set -euo pipefail
src=$1
dst=$2
fps=$3
scale_width=$4
ffmpeg -hide_banner -loglevel error -y \
-i "${src}" \
-vf "fps=${fps},scale='min(${scale_width},iw)':-2" \
-c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \
-c:a pcm_s16le \
"${dst}"
REMOTE_NORMALIZE_SCRIPT
if [[ "${normalize_status}" -ne 0 ]]; then
echo "remote CFR normalization failed; falling back to raw capture" >&2
remote_fetch_capture="${REMOTE_CAPTURE}"
fi
fi
if [[ "${REMOTE_ANALYZE}" != "0" ]]; then
if [[ "${REMOTE_ANALYZE_COPY}" != "0" ]]; then
echo "==> copying sync analyzer to ${TETHYS_HOST}:${REMOTE_ANALYZE_BIN}"
scp ${SSH_OPTS} "${ANALYZE_BIN}" "${TETHYS_HOST}:${REMOTE_ANALYZE_BIN}"
fi
echo "==> analyzing capture on ${TETHYS_HOST}"
ssh ${SSH_OPTS} "${TETHYS_HOST}" \
"chmod +x '${REMOTE_ANALYZE_BIN}' && '${REMOTE_ANALYZE_BIN}' '${remote_fetch_capture}' --json" \
> "${LOCAL_ANALYSIS_JSON}"
fi
if [[ "${FETCH_CAPTURE}" != "0" ]]; then
echo "==> fetching capture back to ${LOCAL_CAPTURE}"
scp ${SSH_OPTS} "${TETHYS_HOST}:${remote_fetch_capture}" "${LOCAL_CAPTURE}"
fi
fi
if [[ "${probe_status}" -ne 0 ]]; then
echo "sync probe failed with status ${probe_status}" >&2
[[ -f "${LOCAL_CAPTURE}" ]] && echo "partial capture preserved at ${LOCAL_CAPTURE}" >&2
exit "${probe_status}"
fi
if [[ "${capture_status}" -ne 0 ]]; then
if [[ "${capture_status}" -eq 141 && ( -f "${LOCAL_CAPTURE}" || -f "${LOCAL_ANALYSIS_JSON}" ) ]]; then
echo "Tethys capture ended with PipeWire SIGPIPE after ffmpeg closed; accepting preserved analysis artifacts" >&2
elif [[ "${capture_status}" -eq 124 && ( -f "${LOCAL_CAPTURE}" || -f "${LOCAL_ANALYSIS_JSON}" ) ]]; then
echo "Tethys capture timed out after preserving analysis artifacts; accepting the run for analysis" >&2
else
echo "Tethys capture failed with status ${capture_status}" >&2
[[ -f "${LOCAL_CAPTURE}" ]] && echo "partial capture preserved at ${LOCAL_CAPTURE}" >&2
exit "${capture_status}"
fi
fi
if [[ "${REMOTE_ANALYZE}" != "0" ]]; then
if [[ ! -f "${LOCAL_ANALYSIS_JSON}" ]]; then
echo "remote analysis did not produce ${LOCAL_ANALYSIS_JSON}" >&2
exit 92
fi
echo "==> remote analysis summary"
python - <<'PY' "${LOCAL_ANALYSIS_JSON}"
import json
import pathlib
import sys
report = json.loads(pathlib.Path(sys.argv[1]).read_text())
print(f"A/V sync report for {sys.argv[1]}")
print(f"- video onsets: {report['video_event_count']}")
print(f"- audio onsets: {report['audio_event_count']}")
print(f"- paired pulses: {report['paired_event_count']}")
print(f"- first skew: {report['first_skew_ms']:+.1f} ms (audio after video is positive)")
print(f"- last skew: {report['last_skew_ms']:+.1f} ms")
print(f"- mean skew: {report['mean_skew_ms']:+.1f} ms")
print(f"- median skew: {report['median_skew_ms']:+.1f} ms")
print(f"- max abs skew: {report['max_abs_skew_ms']:.1f} ms")
print(f"- drift: {report['drift_ms']:+.1f} ms")
cal = report.get('calibration', {})
print(f"- calibration ready: {cal.get('ready')}")
print(f"- recommended audio offset adjust: {int(cal.get('recommended_audio_offset_adjust_us', 0)):+d} us")
print(f"- alternative video offset adjust: {int(cal.get('recommended_video_offset_adjust_us', 0)):+d} us")
print(f"- calibration note: {cal.get('note', '')}")
PY
else
if [[ ! -f "${LOCAL_CAPTURE}" ]]; then
echo "capture was not fetched and REMOTE_ANALYZE=0 left nothing local to analyze" >&2
exit 93
fi
echo "==> analyzing capture"
(
cd "${REPO_ROOT}"
"${ANALYZE_BIN}" "${LOCAL_CAPTURE}"
)
fi
if [[ "${capture_v4l2_fault}" -eq 1 ]]; then
echo "warning: Tethys video capture reported VIDIOC_QBUF / Bad file descriptor; treat unstable skew or analyzer failures as host-capture suspect" >&2
fi
echo "==> done"
if [[ -f "${LOCAL_CAPTURE}" ]]; then
echo "capture: ${LOCAL_CAPTURE}"
fi
if [[ -f "${LOCAL_ANALYSIS_JSON}" ]]; then
echo "analysis_json: ${LOCAL_ANALYSIS_JSON}"
fi
if [[ -f "${LOCAL_CAPTURE_LOG}" ]]; then
echo "capture_log: ${LOCAL_CAPTURE_LOG}"
fi