bstein/lesavka
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
lesavka/scripts/manual/run_upstream_av_sync.sh

952 lines
31 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# scripts/manual/run_upstream_av_sync.sh
# Manual: upstream A/V sync hardware probe; not part of CI.
#
# Manual: capture the real Tethys webcam/mic endpoints while the shared-clock
# sync probe streams upstream media through Lesavka, then analyze the skew.
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"
TETHYS_HOST=${TETHYS_HOST:-tethys}
LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST:-theia}
LESAVKA_SERVER_CONNECT_HOST=${LESAVKA_SERVER_CONNECT_HOST:-38.28.125.112}
LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-auto}
LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https}
LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}
PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-10}
PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4}
PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + 20))}
# Do not open the UVC host capture far ahead of the probe. The gadget side only
# has frames once the sync probe is feeding the server, and some hosts time out
# VIDIOC_STREAMON if the camera is starved during pre-roll.
LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-0}
TAIL_SECONDS=${TAIL_SECONDS:-2}
CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))}
LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}
REMOTE_VIDEO_DEVICE=${REMOTE_VIDEO_DEVICE:-auto}
VIDEO_SIZE=${VIDEO_SIZE:-auto}
VIDEO_FPS=${VIDEO_FPS:-auto}
VIDEO_FORMAT=${VIDEO_FORMAT:-mjpeg}
REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse}
REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-ffmpeg}
REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-copy}
REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto}
REMOTE_AUDIO_QUIESCE_USER_AUDIO=${REMOTE_AUDIO_QUIESCE_USER_AUDIO:-auto}
ANALYSIS_NORMALIZE=${ANALYSIS_NORMALIZE:-0}
ANALYSIS_SCALE_WIDTH=${ANALYSIS_SCALE_WIDTH:-1280}
SSH_OPTS=${SSH_OPTS:-"-o BatchMode=yes -o ConnectTimeout=30"}
LOCAL_AUDIO_SANITY=${LOCAL_AUDIO_SANITY:-1}
PROBE_PREBUILD=${PROBE_PREBUILD:-1}
PROBE_BIN=${PROBE_BIN:-"${REPO_ROOT}/target/debug/lesavka-sync-probe"}
ANALYZE_BIN=${ANALYZE_BIN:-"${REPO_ROOT}/target/debug/lesavka-sync-analyze"}
REMOTE_ANALYZE=${REMOTE_ANALYZE:-1}
REMOTE_ANALYZE_BIN=${REMOTE_ANALYZE_BIN:-/tmp/lesavka-sync-analyze}
REMOTE_ANALYZE_COPY=${REMOTE_ANALYZE_COPY:-1}
FETCH_CAPTURE=${FETCH_CAPTURE:-0}
REMOTE_SERVER_PREFLIGHT=${REMOTE_SERVER_PREFLIGHT:-1}
REMOTE_EXPECT_CAM_OUTPUT=${REMOTE_EXPECT_CAM_OUTPUT:-uvc}
REMOTE_EXPECT_UVC_CODEC=${REMOTE_EXPECT_UVC_CODEC:-mjpeg}
CAPTURE_READY_MARKER="__LESAVKA_CAPTURE_READY__"
STAMP="$(date +%Y%m%d-%H%M%S)"
REMOTE_CAPTURE=${REMOTE_CAPTURE:-"/tmp/lesavka-sync-probe-${STAMP}.mkv"}
LOCAL_REPORT_DIR="${LOCAL_OUTPUT_DIR%/}/lesavka-sync-probe-${STAMP}"
LOCAL_CAPTURE="${LOCAL_REPORT_DIR}/capture.mkv"
LOCAL_ANALYSIS_JSON="${LOCAL_REPORT_DIR}/report.json"
LOCAL_REPORT_TXT="${LOCAL_REPORT_DIR}/report.txt"
LOCAL_EVENTS_CSV="${LOCAL_REPORT_DIR}/events.csv"
LOCAL_CAPTURE_LOG="${LOCAL_REPORT_DIR}/capture.log"
mkdir -p "${LOCAL_REPORT_DIR}"
RESOLVED_LESAVKA_SERVER_ADDR=""
SERVER_TUNNEL_PID=""
SERVER_TUNNEL_REMOTE_PORT=""
SERVER_TUNNEL_LOCAL_PORT=""
cleanup_server_tunnel() {
if [[ -z "${SERVER_TUNNEL_PID}" ]]; then
return 0
fi
if kill -0 "${SERVER_TUNNEL_PID}" >/dev/null 2>&1; then
kill "${SERVER_TUNNEL_PID}" >/dev/null 2>&1 || true
wait "${SERVER_TUNNEL_PID}" >/dev/null 2>&1 || true
fi
}
trap cleanup_server_tunnel EXIT
pick_local_server_tunnel_port() {
python3 - <<'PY'
import socket
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
wait_for_server_tunnel() {
local local_port=$1
local tries=50
local i=0
while (( i < tries )); do
if nc -z 127.0.0.1 "${local_port}" >/dev/null 2>&1; then
return 0
fi
if ! kill -0 "${SERVER_TUNNEL_PID}" >/dev/null 2>&1; then
wait "${SERVER_TUNNEL_PID}" >/dev/null 2>&1 || true
echo "SSH tunnel to ${LESAVKA_SERVER_HOST} exited before becoming ready" >&2
exit 88
fi
sleep 0.1
((i += 1))
done
echo "SSH tunnel to ${LESAVKA_SERVER_HOST} did not become ready on localhost:${local_port}" >&2
exit 89
}
start_server_tunnel() {
local remote_port=$1
local local_port
local_port="$(pick_local_server_tunnel_port)"
echo "==> opening SSH tunnel to ${LESAVKA_SERVER_HOST}:127.0.0.1:${remote_port} on localhost:${local_port}"
ssh ${SSH_OPTS} -o ExitOnForwardFailure=yes \
-N \
-L "127.0.0.1:${local_port}:127.0.0.1:${remote_port}" \
"${LESAVKA_SERVER_HOST}" &
SERVER_TUNNEL_PID=$!
SERVER_TUNNEL_REMOTE_PORT="${remote_port}"
SERVER_TUNNEL_LOCAL_PORT="${local_port}"
wait_for_server_tunnel "${local_port}"
}
resolve_server_addr() {
if [[ "${LESAVKA_SERVER_ADDR}" != "auto" ]]; then
RESOLVED_LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_ADDR}"
return 0
fi
local bind_addr port
bind_addr="$(
ssh ${SSH_OPTS} "${LESAVKA_SERVER_HOST}" \
"grep -E '^LESAVKA_SERVER_BIND_ADDR=' /etc/lesavka/server.env 2>/dev/null | tail -n1 | cut -d= -f2-" \
2>/dev/null || true
)"
port="${bind_addr##*:}"
if [[ "${port}" =~ ^[0-9]+$ ]]; then
start_server_tunnel "${port}"
RESOLVED_LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_SCHEME}://127.0.0.1:${SERVER_TUNNEL_LOCAL_PORT}"
return 0
fi
start_server_tunnel "50051"
RESOLVED_LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_SCHEME}://127.0.0.1:${SERVER_TUNNEL_LOCAL_PORT}"
}
preflight_server_path() {
[[ "${REMOTE_SERVER_PREFLIGHT}" != "0" ]] || return 0
echo "==> verifying Lesavka server path on ${LESAVKA_SERVER_HOST}"
ssh ${SSH_OPTS} "${LESAVKA_SERVER_HOST}" bash -s -- \
"${REMOTE_EXPECT_CAM_OUTPUT}" \
"${REMOTE_EXPECT_UVC_CODEC}" <<'REMOTE_PREFLIGHT'
set -euo pipefail
expect_cam_output=$1
expect_uvc_codec=$2
read_env_value() {
local key=$1
local file=$2
local value=""
value=$(grep -E "^${key}=" "$file" 2>/dev/null | tail -n1 | cut -d= -f2- || true)
printf '%s\n' "$value"
}
cam_output=$(read_env_value "LESAVKA_CAM_OUTPUT" /etc/lesavka/server.env)
server_uvc_codec=$(read_env_value "LESAVKA_UVC_CODEC" /etc/lesavka/server.env)
runtime_uvc_codec=$(read_env_value "LESAVKA_UVC_CODEC" /etc/lesavka/uvc.env)
printf ' ↪ server.env CAM_OUTPUT=%s\n' "${cam_output:-<unset>}"
printf ' ↪ server.env UVC_CODEC=%s\n' "${server_uvc_codec:-<unset>}"
printf ' ↪ uvc.env UVC_CODEC=%s\n' "${runtime_uvc_codec:-<unset>}"
if [[ -n "${expect_cam_output}" && "${cam_output}" != "${expect_cam_output}" ]]; then
printf 'expected CAM_OUTPUT=%s but found %s\n' "${expect_cam_output}" "${cam_output:-<unset>}" >&2
exit 64
fi
if [[ -n "${expect_uvc_codec}" && "${server_uvc_codec}" != "${expect_uvc_codec}" ]]; then
printf 'expected server.env UVC_CODEC=%s but found %s\n' "${expect_uvc_codec}" "${server_uvc_codec:-<unset>}" >&2
exit 65
fi
if [[ -n "${expect_uvc_codec}" && "${runtime_uvc_codec}" != "${expect_uvc_codec}" ]]; then
printf 'expected uvc.env UVC_CODEC=%s but found %s\n' "${expect_uvc_codec}" "${runtime_uvc_codec:-<unset>}" >&2
exit 66
fi
systemctl is-active lesavka-server lesavka-uvc lesavka-core >/dev/null
REMOTE_PREFLIGHT
}
if [[ "${LOCAL_AUDIO_SANITY}" != "0" ]]; then
echo "==> verifying local speaker-to-mic sanity before upstream sync run"
"${SCRIPT_DIR}/run_local_audio_sanity.sh"
fi
if [[ "${PROBE_PREBUILD}" != "0" ]]; then
echo "==> prebuilding sync probe/analyzer before opening the capture window"
(
cd "${REPO_ROOT}"
cargo build -p lesavka_client --bin lesavka-sync-probe --bin lesavka-sync-analyze
)
fi
if [[ ! -x "${PROBE_BIN}" ]]; then
echo "sync probe binary not found at ${PROBE_BIN}" >&2
exit 1
fi
if [[ ! -x "${ANALYZE_BIN}" ]]; then
echo "sync analyzer binary not found at ${ANALYZE_BIN}" >&2
exit 1
fi
resolve_server_addr
echo "==> resolved Lesavka server addr: ${RESOLVED_LESAVKA_SERVER_ADDR}"
if [[ -n "${SERVER_TUNNEL_PID}" ]]; then
echo " ↪ tunneled to ${LESAVKA_SERVER_HOST}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT}"
fi
preflight_server_path
echo "==> starting Tethys capture on ${TETHYS_HOST}"
ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
"${REMOTE_CAPTURE}" \
"${CAPTURE_SECONDS}" \
"${REMOTE_VIDEO_DEVICE}" \
"${VIDEO_SIZE}" \
"${VIDEO_FPS}" \
"${VIDEO_FORMAT}" \
"${REMOTE_CAPTURE_STACK}" \
"${REMOTE_PULSE_CAPTURE_TOOL}" \
"${REMOTE_PULSE_VIDEO_MODE}" \
"${REMOTE_AUDIO_SOURCE}" \
"${REMOTE_AUDIO_QUIESCE_USER_AUDIO}" \
> >(tee "${LOCAL_CAPTURE_LOG}") \
2> >(tee -a "${LOCAL_CAPTURE_LOG}" >&2) <<'REMOTE_CAPTURE_SCRIPT' &
set -euo pipefail
remote_capture=$1
capture_seconds=$2
remote_video_device=$3
video_size=$4
video_fps=$5
video_format=$6
remote_capture_stack=$7
remote_pulse_capture_tool=$8
remote_pulse_video_mode=$9
remote_audio_source=${10}
remote_audio_quiesce_user_audio=${11}
rm -f "${remote_capture}"
restore_user_audio() {
systemctl --user start pipewire.socket pipewire-pulse.socket wireplumber.service >/dev/null 2>&1 || true
sleep 1
systemctl --user start pipewire.service pipewire-pulse.service >/dev/null 2>&1 || true
}
quiesce_user_audio() {
systemctl --user stop pipewire-pulse.service pipewire.service wireplumber.service \
pipewire-pulse.socket pipewire.socket >/dev/null 2>&1 || true
sleep 1
}
resolve_video_device() {
local requested=$1
if [[ "${requested}" != "auto" ]]; then
printf '%s\n' "${requested}"
return 0
fi
local by_id
by_id=$(find /dev/v4l/by-id -maxdepth 1 -type l -name '*Lesavka*video-index0' 2>/dev/null | head -n1 || true)
if [[ -n "${by_id}" ]]; then
printf '%s\n' "${by_id}"
return 0
fi
if command -v v4l2-ctl >/dev/null 2>&1; then
local resolved
resolved="$(
v4l2-ctl --list-devices 2>/dev/null \
| awk '
BEGIN { want=0 }
/Lesavka Composite: UVC Camera/ { want=1; next }
/^[^ \t]/ { want=0 }
want && /^[ \t]+\/dev\/video[0-9]+/ {
gsub(/^[ \t]+/, "", $0)
print
exit
}
'
)"
if [[ -n "${resolved}" ]]; then
printf '%s\n' "${resolved}"
return 0
fi
fi
printf 'Lesavka UVC video device not found on Tethys; refusing to fall back to an unrelated webcam/capture card.\n' >&2
exit 64
}
resolve_pulse_source() {
if ! command -v pactl >/dev/null 2>&1; then
return 1
fi
pactl list short sources 2>/dev/null \
| awk '
/alsa_input\..*Lesavka_Composite/ { print $2; found=1; exit }
/Lesavka_Composite/ && !fallback { fallback=$2 }
END {
if (found) exit 0
if (fallback != "") { print fallback; exit 0 }
exit 1
}
'
}
resolve_alsa_audio_device() {
if ! command -v arecord >/dev/null 2>&1; then
return 1
fi
arecord -l 2>/dev/null | awk '
/^card [0-9]+:/ && ($0 ~ /Lesavka|UAC2_Gadget|UAC2Gadget|Composite/) {
card=$2
sub(":", "", card)
for (i = 1; i <= NF; i++) {
if ($i == "device") {
dev=$(i + 1)
sub(":", "", dev)
printf "hw:%s,%s\n", card, dev
found=1
exit 0
}
}
}
END {
if (!found) exit 1
}
'
}
gst_video_source_caps() {
case "${video_format}" in
""|mjpeg|MJPG)
printf 'image/jpeg,width=%s,height=%s,framerate=%s/1' \
"${resolved_video_size%x*}" \
"${resolved_video_size#*x}" \
"${resolved_video_fps}"
;;
yuyv422|YUYV|yuyv)
printf 'video/x-raw,format=YUY2,width=%s,height=%s,framerate=%s/1' \
"${resolved_video_size%x*}" \
"${resolved_video_size#*x}" \
"${resolved_video_fps}"
;;
*)
printf 'unsupported gst video_format=%s\n' "${video_format}" >&2
exit 64
;;
esac
}
gst_video_decode_chain() {
case "${video_format}" in
""|mjpeg|MJPG)
printf 'jpegdec ! '
;;
yuyv422|YUYV|yuyv)
printf ''
;;
*)
printf 'unsupported gst video_format=%s\n' "${video_format}" >&2
exit 64
;;
esac
}
current_video_profile() {
if ! command -v v4l2-ctl >/dev/null 2>&1; then
return 1
fi
v4l2-ctl -d "${resolved_video_device}" --all 2>/dev/null \
| awk '
/Width\/Height[[:space:]]*:/ {
split($0, a, ":")
gsub(/^[ \t]+/, "", a[2])
split(a[2], wh, "/")
width=wh[1]
height=wh[2]
next
}
/Frames per second:/ {
fps=$4
sub(/\..*/, "", fps)
}
END {
if (width != "" && height != "") {
print "size=" width "x" height
}
if (fps != "") {
print "fps=" fps
}
}
'
}
resolve_video_size() {
local requested=$1
if [[ "${requested}" != "auto" ]]; then
printf '%s\n' "${requested}"
return 0
fi
local current_profile
current_profile="$(current_video_profile || true)"
local current_size
current_size="$(awk -F= '/^size=/{print $2; exit}' <<<"${current_profile}")"
if [[ -n "${current_size}" ]]; then
printf '%s\n' "${current_size}"
return 0
fi
if ! command -v v4l2-ctl >/dev/null 2>&1; then
printf '640x480\n'
return 0
fi
local listing
listing="$(v4l2-ctl -d "${resolved_video_device}" --list-formats-ext 2>/dev/null || true)"
local preferred
for preferred in 1920x1080 1360x768 1280x720; do
if grep -q "Size: Discrete ${preferred}" <<<"${listing}"; then
printf '%s\n' "${preferred}"
return 0
fi
done
local first_size
first_size="$(grep -m1 -o 'Size: Discrete [0-9]\+x[0-9]\+' <<<"${listing}" | awk '{print $3}' || true)"
if [[ -n "${first_size}" ]]; then
printf '%s\n' "${first_size}"
return 0
fi
printf '640x480\n'
}
resolve_video_fps() {
local requested=$1
if [[ "${requested}" != "auto" ]]; then
printf '%s\n' "${requested}"
return 0
fi
local current_profile
current_profile="$(current_video_profile || true)"
local current_fps
current_fps="$(awk -F= '/^fps=/{print $2; exit}' <<<"${current_profile}")"
if [[ -n "${current_fps}" ]]; then
printf '%s\n' "${current_fps}"
return 0
fi
if ! command -v v4l2-ctl >/dev/null 2>&1; then
printf '20\n'
return 0
fi
local listing
listing="$(v4l2-ctl -d "${resolved_video_device}" --list-formats-ext 2>/dev/null || true)"
local first_fps
first_fps="$(grep -m1 -o '[0-9]\+\.[0-9]\+ fps' <<<"${listing}" | awk '{sub(/\..*/, "", $1); print $1}' || true)"
if [[ -n "${first_fps}" ]]; then
printf '%s\n' "${first_fps}"
return 0
fi
printf '20\n'
}
resolve_pw_audio_target() {
if ! command -v pw-dump >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then
return 1
fi
pw-dump | python3 -c '
import json
import sys
try:
objs = json.load(sys.stdin)
except Exception:
raise SystemExit(1)
for obj in objs:
if obj.get("type") != "PipeWire:Interface:Node":
continue
props = (obj.get("info") or {}).get("props") or {}
if props.get("media.class") != "Audio/Source":
continue
serial = props.get("object.serial")
name = props.get("node.name", "")
desc = props.get("node.description", "")
if serial is None:
continue
if "Lesavka_Composite" in name or "Lesavka Composite" in desc:
print(serial)
raise SystemExit(0)
raise SystemExit(1)
'
}
capture_mode="alsa"
alsa_audio_dev="hw:3,0"
pulse_source=""
pw_audio_target=""
case "${remote_capture_stack}" in
auto)
if [[ "${remote_audio_source}" == "auto" ]]; then
if pulse_source="$(resolve_pulse_source)"; then
capture_mode="pulse"
elif alsa_audio_dev="$(resolve_alsa_audio_device)"; then
capture_mode="alsa"
printf 'PipeWire Lesavka source not found; falling back to ALSA device %s\n' "${alsa_audio_dev}" >&2
elif command -v pw-record >/dev/null 2>&1 \
&& command -v pw-v4l2 >/dev/null 2>&1 \
&& pw_audio_target="$(resolve_pw_audio_target)"; then
capture_mode="pwpipe"
else
printf 'Lesavka audio source not found in PipeWire or ALSA; capture host does not currently expose the gadget microphone.\n' >&2
exit 64
fi
elif [[ "${remote_audio_source}" == pulse:* ]]; then
capture_mode="pulse"
pulse_source="${remote_audio_source#pulse:}"
elif [[ "${remote_audio_source}" == alsa:* ]]; then
alsa_audio_dev="${remote_audio_source#alsa:}"
else
printf 'unsupported REMOTE_AUDIO_SOURCE=%s\n' "${remote_audio_source}" >&2
exit 64
fi
;;
pwpipe)
if ! command -v pw-record >/dev/null 2>&1 || ! command -v pw-v4l2 >/dev/null 2>&1; then
printf 'REMOTE_CAPTURE_STACK=pwpipe requires pw-record and pw-v4l2\n' >&2
exit 64
fi
pw_audio_target="$(resolve_pw_audio_target)" || {
printf 'PipeWire Lesavka capture target not found for REMOTE_CAPTURE_STACK=pwpipe\n' >&2
exit 64
}
capture_mode="pwpipe"
;;
pulse)
if [[ "${remote_audio_source}" == pulse:* ]]; then
pulse_source="${remote_audio_source#pulse:}"
elif [[ "${remote_audio_source}" == "auto" ]]; then
pulse_source="$(resolve_pulse_source)" || {
printf 'PipeWire Lesavka source not found for REMOTE_CAPTURE_STACK=pulse\n' >&2
exit 64
}
else
pulse_source="${remote_audio_source}"
fi
capture_mode="pulse"
;;
alsa)
if [[ "${remote_audio_source}" == alsa:* ]]; then
alsa_audio_dev="${remote_audio_source#alsa:}"
elif [[ "${remote_audio_source}" != "auto" ]]; then
alsa_audio_dev="${remote_audio_source}"
fi
capture_mode="alsa"
;;
*)
printf 'unsupported REMOTE_CAPTURE_STACK=%s\n' "${remote_capture_stack}" >&2
exit 64
;;
esac
resolved_video_device="$(resolve_video_device "${remote_video_device}")"
resolved_video_size="$(resolve_video_size "${video_size}")"
resolved_video_fps="$(resolve_video_fps "${video_fps}")"
printf 'using video device: %s\n' "${resolved_video_device}" >&2
printf 'using video mode: %s @ %s fps (%s)\n' "${resolved_video_size}" "${resolved_video_fps}" "${video_format:-driver-default}" >&2
printf '%s\n' "__LESAVKA_CAPTURE_READY__"
video_args=(-f video4linux2 -framerate "${resolved_video_fps}" -video_size "${resolved_video_size}")
if [[ -n "${video_format}" ]]; then
video_args+=(-input_format "${video_format}")
fi
gst_source_caps="$(gst_video_source_caps)"
gst_decode_chain="$(gst_video_decode_chain)"
run_ffmpeg_capture() {
local rc=0
timeout --signal=INT "$((capture_seconds + 5))" "$@" || rc=$?
case "${rc}" in
0|124|130)
return 0
;;
*)
return "${rc}"
;;
esac
}
quiesce_for_alsa=0
case "${remote_audio_quiesce_user_audio}" in
1|true|yes)
quiesce_for_alsa=1
;;
auto)
if [[ "${capture_mode}" == "alsa" ]]; then
quiesce_for_alsa=1
fi
;;
0|false|no)
quiesce_for_alsa=0
;;
*)
printf 'unsupported REMOTE_AUDIO_QUIESCE_USER_AUDIO=%s\n' "${remote_audio_quiesce_user_audio}" >&2
exit 64
;;
esac
if [[ "${capture_mode}" == "alsa" && "${quiesce_for_alsa}" == "1" ]]; then
printf 'quiescing Tethys user audio before raw ALSA capture\n' >&2
quiesce_user_audio
trap restore_user_audio EXIT
fi
if [[ "${capture_mode}" == "pwpipe" ]]; then
printf 'using PipeWire-native mux capture target serial: %s\n' "${pw_audio_target}" >&2
timeout "${capture_seconds}" pw-record \
--target "${pw_audio_target}" \
--rate 48000 \
--channels 2 \
--format s16 \
--raw - \
| pw-v4l2 ffmpeg -hide_banner -loglevel error -y \
-thread_queue_size 1024 \
"${video_args[@]}" \
-i "${resolved_video_device}" \
-thread_queue_size 1024 \
-f s16le -ar 48000 -ac 2 \
-i pipe:0 \
-t "${capture_seconds}" \
-c:v copy \
-c:a pcm_s16le \
"${remote_capture}"
elif [[ "${capture_mode}" == "pulse" ]]; then
printf 'using Pulse source: %s\n' "${pulse_source}" >&2
case "${remote_pulse_capture_tool}" in
ffmpeg)
case "${remote_pulse_video_mode}" in
copy)
run_ffmpeg_capture ffmpeg -hide_banner -loglevel error -y \
-thread_queue_size 1024 \
"${video_args[@]}" \
-i "${resolved_video_device}" \
-thread_queue_size 1024 \
-f pulse \
-i "${pulse_source}" \
-t "${capture_seconds}" \
-c:v copy \
-c:a pcm_s16le \
"${remote_capture}"
;;
cfr)
run_ffmpeg_capture ffmpeg -hide_banner -loglevel error -y \
-thread_queue_size 1024 \
"${video_args[@]}" \
-i "${resolved_video_device}" \
-thread_queue_size 1024 \
-f pulse \
-i "${pulse_source}" \
-t "${capture_seconds}" \
-vf "fps=${resolved_video_fps}" \
-c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \
-c:a pcm_s16le \
"${remote_capture}"
;;
*)
printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2
exit 64
;;
esac
;;
gst)
case "${remote_pulse_video_mode}" in
copy)
if [[ "${video_format}" != "mjpeg" && "${video_format}" != "MJPG" && -n "${video_format}" ]]; then
printf 'gst copy mode only supports mjpeg input, got %s\n' "${video_format}" >&2
exit 64
fi
timeout --signal=INT "$((capture_seconds + 3))" \
gst-launch-1.0 -q -e \
matroskamux name=mux ! filesink location="${remote_capture}" \
v4l2src device="${resolved_video_device}" do-timestamp=true ! \
${gst_source_caps} ! \
queue ! mux. \
pulsesrc device="${pulse_source}" do-timestamp=true ! \
audio/x-raw,rate=48000,channels=2 ! \
audioconvert ! audioresample ! queue ! mux. || true
;;
cfr)
timeout --signal=INT "$((capture_seconds + 3))" \
gst-launch-1.0 -q -e \
matroskamux name=mux ! filesink location="${remote_capture}" \
v4l2src device="${resolved_video_device}" do-timestamp=true ! \
${gst_source_caps} ! \
${gst_decode_chain} \
videoconvert ! videorate ! video/x-raw,framerate="${video_fps}"/1 ! \
x264enc tune=zerolatency speed-preset=ultrafast key-int-max=1 bitrate=5000 ! \
h264parse ! \
queue ! mux. \
pulsesrc device="${pulse_source}" do-timestamp=true ! \
audio/x-raw,rate=48000,channels=2 ! \
audioconvert ! audioresample ! queue ! mux. || true
;;
*)
printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2
exit 64
;;
esac
;;
*)
printf 'unsupported REMOTE_PULSE_CAPTURE_TOOL=%s\n' "${remote_pulse_capture_tool}" >&2
exit 64
;;
esac
else
run_ffmpeg_capture ffmpeg -hide_banner -loglevel error -y \
-thread_queue_size 1024 \
"${video_args[@]}" \
-i "${resolved_video_device}" \
-thread_queue_size 1024 \
-f alsa -ac 2 -ar 48000 \
-i "${alsa_audio_dev}" \
-t "${capture_seconds}" \
-c:v ffv1 -level 3 -g 1 \
-c:a pcm_s16le \
"${remote_capture}"
fi
REMOTE_CAPTURE_SCRIPT
capture_pid=$!
wait_for_capture_ready() {
local tries=100
local i=0
while (( i < tries )); do
if [[ -f "${LOCAL_CAPTURE_LOG}" ]] && grep -q "${CAPTURE_READY_MARKER}" "${LOCAL_CAPTURE_LOG}"; then
return 0
fi
if ! kill -0 "${capture_pid}" >/dev/null 2>&1; then
capture_status=0
wait "${capture_pid}" || capture_status=$?
echo "Tethys capture failed before the sync probe could start; see ${LOCAL_CAPTURE_LOG} for details." >&2
exit "${capture_status}"
fi
sleep 0.1
((i += 1))
done
echo "Timed out waiting for Tethys capture to become ready; see ${LOCAL_CAPTURE_LOG} for details." >&2
exit 90
}
wait_for_capture_ready
sleep "${LEAD_IN_SECONDS}"
echo "==> running local Lesavka sync probe against ${RESOLVED_LESAVKA_SERVER_ADDR}"
probe_status=0
probe_timed_out=0
(
cd "${REPO_ROOT}"
LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \
timeout --signal=INT "${PROBE_TIMEOUT_SECONDS}" "${PROBE_BIN}" \
--server "${RESOLVED_LESAVKA_SERVER_ADDR}" \
--duration-seconds "${PROBE_DURATION_SECONDS}" \
--warmup-seconds "${PROBE_WARMUP_SECONDS}"
) || probe_status=$?
if [[ "${probe_status}" -eq 124 ]]; then
probe_timed_out=1
fi
capture_status=0
wait "${capture_pid}" || capture_status=$?
capture_v4l2_fault=0
if [[ -f "${LOCAL_CAPTURE_LOG}" ]] \
&& grep -q 'VIDIOC_QBUF): Bad file descriptor' "${LOCAL_CAPTURE_LOG}"; then
capture_v4l2_fault=1
fi
capture_streamon_timeout=0
if [[ -f "${LOCAL_CAPTURE_LOG}" ]] \
&& grep -q 'VIDIOC_STREAMON.*Connection timed out' "${LOCAL_CAPTURE_LOG}"; then
capture_streamon_timeout=1
fi
if ssh ${SSH_OPTS} "${TETHYS_HOST}" "test -f '${REMOTE_CAPTURE}'"; then
remote_fetch_capture="${REMOTE_CAPTURE}"
if [[ "${ANALYSIS_NORMALIZE}" != "0" ]]; then
remote_fetch_capture="${REMOTE_CAPTURE%.mkv}-analysis.mkv"
echo "==> normalizing remote capture to CFR for analysis"
normalize_status=0
ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
"${REMOTE_CAPTURE}" \
"${remote_fetch_capture}" \
"${VIDEO_FPS}" \
"${ANALYSIS_SCALE_WIDTH}" <<'REMOTE_NORMALIZE_SCRIPT' || normalize_status=$?
set -euo pipefail
src=$1
dst=$2
fps=$3
scale_width=$4
ffmpeg -hide_banner -loglevel error -y \
-i "${src}" \
-vf "fps=${fps},scale='min(${scale_width},iw)':-2" \
-c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \
-c:a pcm_s16le \
"${dst}"
REMOTE_NORMALIZE_SCRIPT
if [[ "${normalize_status}" -ne 0 ]]; then
echo "remote CFR normalization failed; falling back to raw capture" >&2
remote_fetch_capture="${REMOTE_CAPTURE}"
fi
fi
if [[ "${REMOTE_ANALYZE}" != "0" ]]; then
if [[ "${REMOTE_ANALYZE_COPY}" != "0" ]]; then
echo "==> copying sync analyzer to ${TETHYS_HOST}:${REMOTE_ANALYZE_BIN}"
scp ${SSH_OPTS} "${ANALYZE_BIN}" "${TETHYS_HOST}:${REMOTE_ANALYZE_BIN}"
fi
echo "==> analyzing capture on ${TETHYS_HOST}"
ssh ${SSH_OPTS} "${TETHYS_HOST}" \
"chmod +x '${REMOTE_ANALYZE_BIN}' && '${REMOTE_ANALYZE_BIN}' '${remote_fetch_capture}' --json" \
> "${LOCAL_ANALYSIS_JSON}"
fi
if [[ "${FETCH_CAPTURE}" != "0" ]]; then
echo "==> fetching capture back to ${LOCAL_CAPTURE}"
scp ${SSH_OPTS} "${TETHYS_HOST}:${remote_fetch_capture}" "${LOCAL_CAPTURE}"
fi
fi
if [[ "${probe_status}" -ne 0 ]]; then
if [[ "${probe_timed_out}" -eq 1 ]]; then
echo "sync probe timed out after ${PROBE_TIMEOUT_SECONDS}s; this usually means one upstream stream did not close cleanly after capture starvation." >&2
fi
echo "sync probe failed with status ${probe_status}" >&2
[[ -f "${LOCAL_CAPTURE}" ]] && echo "partial capture preserved at ${LOCAL_CAPTURE}" >&2
exit "${probe_status}"
fi
if [[ "${capture_status}" -ne 0 ]]; then
if [[ "${capture_streamon_timeout}" -eq 1 ]]; then
echo "Tethys capture timed out during VIDIOC_STREAMON; the UVC host opened before MJPEG frames reached the gadget." >&2
echo "Keep LEAD_IN_SECONDS=0 and restart lesavka-uvc/lesavka-server before retrying if the gadget is wedged from an earlier failed run." >&2
fi
if [[ "${capture_status}" -eq 141 && ( -f "${LOCAL_CAPTURE}" || -f "${LOCAL_ANALYSIS_JSON}" ) ]]; then
echo "Tethys capture ended with PipeWire SIGPIPE after ffmpeg closed; accepting preserved analysis artifacts" >&2
elif [[ "${capture_status}" -eq 124 && ( -f "${LOCAL_CAPTURE}" || -f "${LOCAL_ANALYSIS_JSON}" ) ]]; then
echo "Tethys capture timed out after preserving analysis artifacts; accepting the run for analysis" >&2
else
echo "Tethys capture failed with status ${capture_status}" >&2
[[ -f "${LOCAL_CAPTURE}" ]] && echo "partial capture preserved at ${LOCAL_CAPTURE}" >&2
exit "${capture_status}"
fi
fi
if [[ "${REMOTE_ANALYZE}" != "0" ]]; then
if [[ ! -f "${LOCAL_ANALYSIS_JSON}" ]]; then
echo "remote analysis did not produce ${LOCAL_ANALYSIS_JSON}" >&2
exit 92
fi
echo "==> remote analysis summary"
python - <<'PY' "${LOCAL_ANALYSIS_JSON}" "${LOCAL_REPORT_TXT}" "${LOCAL_EVENTS_CSV}"
import csv
import json
import pathlib
import sys
report = json.loads(pathlib.Path(sys.argv[1]).read_text())
verdict = report.get('verdict', {})
cal = report.get('calibration', {})
lines = [
f"A/V sync report for {sys.argv[1]}",
f"- verdict: {verdict.get('status', 'unknown')} ({'pass' if verdict.get('passed') else 'fail'})",
f"- verdict reason: {verdict.get('reason', '')}",
f"- p95 abs skew: {float(verdict.get('p95_abs_skew_ms', 0.0)):.1f} ms",
f"- video onsets: {report['video_event_count']}",
f"- audio onsets: {report['audio_event_count']}",
f"- paired pulses: {report['paired_event_count']}",
f"- first skew: {report['first_skew_ms']:+.1f} ms (audio after video is positive)",
f"- last skew: {report['last_skew_ms']:+.1f} ms",
f"- mean skew: {report['mean_skew_ms']:+.1f} ms",
f"- median skew: {report['median_skew_ms']:+.1f} ms",
f"- max abs skew: {report['max_abs_skew_ms']:.1f} ms",
f"- drift: {report['drift_ms']:+.1f} ms",
f"- calibration ready: {cal.get('ready')}",
f"- recommended audio offset adjust: {int(cal.get('recommended_audio_offset_adjust_us', 0)):+d} us",
f"- alternative video offset adjust: {int(cal.get('recommended_video_offset_adjust_us', 0)):+d} us",
f"- calibration note: {cal.get('note', '')}",
]
summary = "\n".join(lines) + "\n"
pathlib.Path(sys.argv[2]).write_text(summary)
with pathlib.Path(sys.argv[3]).open("w", newline="") as handle:
writer = csv.DictWriter(
handle,
fieldnames=["event_id", "video_time_s", "audio_time_s", "skew_ms", "confidence"],
)
writer.writeheader()
for event in report.get("paired_events", []):
writer.writerow({
"event_id": event.get("event_id"),
"video_time_s": event.get("video_time_s"),
"audio_time_s": event.get("audio_time_s"),
"skew_ms": event.get("skew_ms"),
"confidence": event.get("confidence"),
})
print(summary, end="")
PY
else
if [[ ! -f "${LOCAL_CAPTURE}" ]]; then
echo "capture was not fetched and REMOTE_ANALYZE=0 left nothing local to analyze" >&2
exit 93
fi
echo "==> analyzing capture"
(
cd "${REPO_ROOT}"
"${ANALYZE_BIN}" "${LOCAL_CAPTURE}" --report-dir "${LOCAL_REPORT_DIR}"
)
fi
if [[ "${capture_v4l2_fault}" -eq 1 ]]; then
echo "warning: Tethys video capture reported VIDIOC_QBUF / Bad file descriptor; treat unstable skew or analyzer failures as host-capture suspect" >&2
fi
echo "==> done"
echo "artifact_dir: ${LOCAL_REPORT_DIR}"
if [[ -f "${LOCAL_CAPTURE}" ]]; then
echo "capture: ${LOCAL_CAPTURE}"
fi
if [[ -f "${LOCAL_ANALYSIS_JSON}" ]]; then
echo "report_json: ${LOCAL_ANALYSIS_JSON}"
fi
if [[ -f "${LOCAL_REPORT_TXT}" ]]; then
echo "report_txt: ${LOCAL_REPORT_TXT}"
fi
if [[ -f "${LOCAL_EVENTS_CSV}" ]]; then
echo "events_csv: ${LOCAL_EVENTS_CSV}"
fi
if [[ -f "${LOCAL_CAPTURE_LOG}" ]]; then
echo "capture_log: ${LOCAL_CAPTURE_LOG}"
fi
Reference in New Issue View Git Blame Copy Permalink