lesavka/scripts/manual/run_upstream_av_sync.sh

#!/usr/bin/env bash
# scripts/manual/run_upstream_av_sync.sh
#
# Manual: capture the real Tethys webcam/mic endpoints while the shared-clock
# sync probe streams upstream media through Lesavka, then analyze the skew.

set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"

TETHYS_HOST=${TETHYS_HOST:-tethys}
LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-http://38.28.125.112:50051}
PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-10}
PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4}
LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-8}
TAIL_SECONDS=${TAIL_SECONDS:-2}
CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))}
REMOTE_CAPTURE=${REMOTE_CAPTURE:-/tmp/lesavka-upstream-av-sync.mkv}
LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-"${REPO_ROOT}/tmp"}
VIDEO_SIZE=${VIDEO_SIZE:-1280x720}
VIDEO_FPS=${VIDEO_FPS:-30}
VIDEO_FORMAT=${VIDEO_FORMAT:-mjpeg}
REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-auto}
REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto}
REMOTE_AUDIO_QUIESCE_USER_AUDIO=${REMOTE_AUDIO_QUIESCE_USER_AUDIO:-auto}
ANALYSIS_NORMALIZE=${ANALYSIS_NORMALIZE:-1}
SSH_OPTS=${SSH_OPTS:-"-o BatchMode=yes -o ConnectTimeout=5"}
LOCAL_AUDIO_SANITY=${LOCAL_AUDIO_SANITY:-1}
PROBE_PREBUILD=${PROBE_PREBUILD:-1}
PROBE_BIN=${PROBE_BIN:-"${REPO_ROOT}/target/debug/lesavka-sync-probe"}
ANALYZE_BIN=${ANALYZE_BIN:-"${REPO_ROOT}/target/debug/lesavka-sync-analyze"}

mkdir -p "${LOCAL_OUTPUT_DIR}"
STAMP="$(date +%Y%m%d-%H%M%S)"
LOCAL_CAPTURE="${LOCAL_OUTPUT_DIR}/lesavka-upstream-av-sync-${STAMP}.mkv"

if [[ "${LOCAL_AUDIO_SANITY}" != "0" ]]; then
  echo "==> verifying local speaker-to-mic sanity before upstream sync run"
  "${SCRIPT_DIR}/run_local_audio_sanity.sh"
fi

if [[ "${PROBE_PREBUILD}" != "0" ]]; then
  echo "==> prebuilding sync probe/analyzer before opening the capture window"
  (
    cd "${REPO_ROOT}"
    cargo build -p lesavka_client --bin lesavka-sync-probe --bin lesavka-sync-analyze
  )
fi

if [[ ! -x "${PROBE_BIN}" ]]; then
  echo "sync probe binary not found at ${PROBE_BIN}" >&2
  exit 1
fi
if [[ ! -x "${ANALYZE_BIN}" ]]; then
  echo "sync analyzer binary not found at ${ANALYZE_BIN}" >&2
  exit 1
fi

echo "==> starting Tethys capture on ${TETHYS_HOST}"
ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
  "${REMOTE_CAPTURE}" \
  "${CAPTURE_SECONDS}" \
  "${VIDEO_SIZE}" \
  "${VIDEO_FPS}" \
  "${VIDEO_FORMAT}" \
  "${REMOTE_CAPTURE_STACK}" \
  "${REMOTE_AUDIO_SOURCE}" \
  "${REMOTE_AUDIO_QUIESCE_USER_AUDIO}" <<'REMOTE_CAPTURE_SCRIPT' &
set -euo pipefail
remote_capture=$1
capture_seconds=$2
video_size=$3
video_fps=$4
video_format=$5
remote_capture_stack=$6
remote_audio_source=$7
remote_audio_quiesce_user_audio=$8

rm -f "${remote_capture}"
video_args=(-f video4linux2 -framerate "${video_fps}" -video_size "${video_size}")
if [[ -n "${video_format}" ]]; then
  video_args+=(-input_format "${video_format}")
fi

restore_user_audio() {
  systemctl --user start pipewire.socket pipewire-pulse.socket wireplumber.service >/dev/null 2>&1 || true
  sleep 1
  systemctl --user start pipewire.service pipewire-pulse.service >/dev/null 2>&1 || true
}

quiesce_user_audio() {
  systemctl --user stop pipewire-pulse.service pipewire.service wireplumber.service \
    pipewire-pulse.socket pipewire.socket >/dev/null 2>&1 || true
  sleep 1
}

resolve_pulse_source() {
  if ! command -v pactl >/dev/null 2>&1; then
    return 1
  fi
  pactl list short sources 2>/dev/null \
    | awk '
      /alsa_input\..*Lesavka_Composite/ { print $2; found=1; exit }
      /Lesavka_Composite/ && !fallback { fallback=$2 }
      END {
        if (found) exit 0
        if (fallback != "") { print fallback; exit 0 }
        exit 1
      }
    '
}

resolve_pw_audio_target() {
  if ! command -v pw-dump >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then
    return 1
  fi
  pw-dump | python3 - <<'PY'
import json
import sys

try:
    objs = json.load(sys.stdin)
except Exception:
    raise SystemExit(1)

for obj in objs:
    if obj.get("type") != "PipeWire:Interface:Node":
        continue
    props = (obj.get("info") or {}).get("props") or {}
    if props.get("media.class") != "Audio/Source":
        continue
    serial = props.get("object.serial")
    name = props.get("node.name", "")
    desc = props.get("node.description", "")
    if serial is None:
        continue
    if "Lesavka_Composite" in name or "Lesavka Composite" in desc:
        print(serial)
        raise SystemExit(0)

raise SystemExit(1)
PY
}

capture_mode="alsa"
alsa_audio_dev="hw:3,0"
pulse_source=""
pw_audio_target=""

case "${remote_capture_stack}" in
  auto)
    if command -v pw-record >/dev/null 2>&1 \
      && command -v pw-v4l2 >/dev/null 2>&1 \
      && pw_audio_target="$(resolve_pw_audio_target)"; then
      capture_mode="pwpipe"
    elif [[ "${remote_audio_source}" == "auto" ]]; then
      if pulse_source="$(resolve_pulse_source)"; then
        capture_mode="pulse"
      else
        printf 'PipeWire Lesavka source not found; falling back to hw:3,0\n' >&2
      fi
    elif [[ "${remote_audio_source}" == pulse:* ]]; then
      capture_mode="pulse"
      pulse_source="${remote_audio_source#pulse:}"
    elif [[ "${remote_audio_source}" == alsa:* ]]; then
      alsa_audio_dev="${remote_audio_source#alsa:}"
    else
      printf 'unsupported REMOTE_AUDIO_SOURCE=%s\n' "${remote_audio_source}" >&2
      exit 64
    fi
    ;;
  pwpipe)
    if ! command -v pw-record >/dev/null 2>&1 || ! command -v pw-v4l2 >/dev/null 2>&1; then
      printf 'REMOTE_CAPTURE_STACK=pwpipe requires pw-record and pw-v4l2\n' >&2
      exit 64
    fi
    pw_audio_target="$(resolve_pw_audio_target)" || {
      printf 'PipeWire Lesavka capture target not found for REMOTE_CAPTURE_STACK=pwpipe\n' >&2
      exit 64
    }
    capture_mode="pwpipe"
    ;;
  pulse)
    if [[ "${remote_audio_source}" == pulse:* ]]; then
      pulse_source="${remote_audio_source#pulse:}"
    elif [[ "${remote_audio_source}" == "auto" ]]; then
      pulse_source="$(resolve_pulse_source)" || {
        printf 'PipeWire Lesavka source not found for REMOTE_CAPTURE_STACK=pulse\n' >&2
        exit 64
      }
    else
      pulse_source="${remote_audio_source}"
    fi
    capture_mode="pulse"
    ;;
  alsa)
    if [[ "${remote_audio_source}" == alsa:* ]]; then
      alsa_audio_dev="${remote_audio_source#alsa:}"
    elif [[ "${remote_audio_source}" != "auto" ]]; then
      alsa_audio_dev="${remote_audio_source}"
    fi
    capture_mode="alsa"
    ;;
  *)
    printf 'unsupported REMOTE_CAPTURE_STACK=%s\n' "${remote_capture_stack}" >&2
    exit 64
    ;;
esac

quiesce_for_alsa=0
case "${remote_audio_quiesce_user_audio}" in
  1|true|yes)
    quiesce_for_alsa=1
    ;;
  auto)
    if [[ "${audio_mode}" == "alsa" ]]; then
      quiesce_for_alsa=1
    fi
    ;;
  0|false|no)
    quiesce_for_alsa=0
    ;;
  *)
    printf 'unsupported REMOTE_AUDIO_QUIESCE_USER_AUDIO=%s\n' "${remote_audio_quiesce_user_audio}" >&2
    exit 64
    ;;
esac

if [[ "${capture_mode}" == "alsa" && "${quiesce_for_alsa}" == "1" ]]; then
  printf 'quiescing Tethys user audio before raw ALSA capture\n' >&2
  quiesce_user_audio
  trap restore_user_audio EXIT
fi

if [[ "${capture_mode}" == "pwpipe" ]]; then
  printf 'using PipeWire-native mux capture target serial: %s\n' "${pw_audio_target}" >&2
  timeout "${capture_seconds}" pw-record \
    --target "${pw_audio_target}" \
    --rate 48000 \
    --channels 2 \
    --format s16 \
    --raw - \
    | pw-v4l2 ffmpeg -hide_banner -loglevel error -y \
      -thread_queue_size 1024 \
      "${video_args[@]}" \
      -i /dev/video0 \
      -thread_queue_size 1024 \
      -f s16le -ar 48000 -ac 2 \
      -i pipe:0 \
      -t "${capture_seconds}" \
      -c:v copy \
      -c:a pcm_s16le \
      "${remote_capture}"
elif [[ "${capture_mode}" == "pulse" ]]; then
  printf 'using Pulse source: %s\n' "${pulse_source}" >&2
  ffmpeg -hide_banner -loglevel error -y \
    -thread_queue_size 1024 \
    "${video_args[@]}" \
    -i /dev/video0 \
    -thread_queue_size 1024 \
    -f pulse \
    -i "${pulse_source}" \
    -t "${capture_seconds}" \
    -c:v ffv1 -level 3 -g 1 \
    -c:a pcm_s16le \
    "${remote_capture}"
else
  ffmpeg -hide_banner -loglevel error -y \
    -thread_queue_size 1024 \
    "${video_args[@]}" \
    -i /dev/video0 \
    -thread_queue_size 1024 \
    -f alsa -ac 2 -ar 48000 \
    -i "${alsa_audio_dev}" \
    -t "${capture_seconds}" \
    -c:v ffv1 -level 3 -g 1 \
    -c:a pcm_s16le \
    "${remote_capture}"
fi
REMOTE_CAPTURE_SCRIPT
capture_pid=$!

sleep "${LEAD_IN_SECONDS}"

echo "==> running local Lesavka sync probe against ${LESAVKA_SERVER_ADDR}"
probe_status=0
(
  cd "${REPO_ROOT}"
  "${PROBE_BIN}" \
    --server "${LESAVKA_SERVER_ADDR}" \
    --duration-seconds "${PROBE_DURATION_SECONDS}" \
    --warmup-seconds "${PROBE_WARMUP_SECONDS}"
) || probe_status=$?

capture_status=0
wait "${capture_pid}" || capture_status=$?

if ssh ${SSH_OPTS} "${TETHYS_HOST}" "test -f '${REMOTE_CAPTURE}'"; then
  remote_fetch_capture="${REMOTE_CAPTURE}"
  if [[ "${ANALYSIS_NORMALIZE}" != "0" ]]; then
    remote_fetch_capture="${REMOTE_CAPTURE%.mkv}-analysis.mkv"
    echo "==> normalizing remote capture to CFR for analysis"
    normalize_status=0
    ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
      "${REMOTE_CAPTURE}" \
      "${remote_fetch_capture}" \
      "${VIDEO_FPS}" <<'REMOTE_NORMALIZE_SCRIPT' || normalize_status=$?
set -euo pipefail
src=$1
dst=$2
fps=$3
ffmpeg -hide_banner -loglevel error -y \
  -i "${src}" \
  -vf "fps=${fps}" \
  -c:v libx264 -preset ultrafast -crf 12 -g 1 -pix_fmt yuv420p \
  -c:a pcm_s16le \
  "${dst}"
REMOTE_NORMALIZE_SCRIPT
    if [[ "${normalize_status}" -ne 0 ]]; then
      echo "remote CFR normalization failed; falling back to raw capture" >&2
      remote_fetch_capture="${REMOTE_CAPTURE}"
    fi
  fi
  echo "==> fetching capture back to ${LOCAL_CAPTURE}"
  scp ${SSH_OPTS} "${TETHYS_HOST}:${remote_fetch_capture}" "${LOCAL_CAPTURE}"
fi

if [[ "${probe_status}" -ne 0 ]]; then
  echo "sync probe failed with status ${probe_status}" >&2
  [[ -f "${LOCAL_CAPTURE}" ]] && echo "partial capture preserved at ${LOCAL_CAPTURE}" >&2
  exit "${probe_status}"
fi
if [[ "${capture_status}" -ne 0 ]]; then
  if [[ "${capture_status}" -eq 141 && -f "${LOCAL_CAPTURE}" ]]; then
    echo "Tethys capture ended with PipeWire SIGPIPE after ffmpeg closed; accepting preserved capture ${LOCAL_CAPTURE}" >&2
  else
  echo "Tethys capture failed with status ${capture_status}" >&2
  [[ -f "${LOCAL_CAPTURE}" ]] && echo "partial capture preserved at ${LOCAL_CAPTURE}" >&2
  exit "${capture_status}"
  fi
fi

echo "==> analyzing capture"
(
  cd "${REPO_ROOT}"
  "${ANALYZE_BIN}" "${LOCAL_CAPTURE}"
)

echo "==> done"
echo "capture: ${LOCAL_CAPTURE}"