lesavka/scripts/manual/run_client_to_rct_transport_probe.sh

351 lines
14 KiB
Bash
Executable File

#!/usr/bin/env bash
# scripts/manual/run_client_to_rct_transport_probe.sh
# Manual hardware probe: inject bundled synthetic client media, then measure
# final RCT UVC/UAC sync, freshness, and smoothness without mutating the server.
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"
TETHYS_HOST=${TETHYS_HOST:-tethys}
LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST:-theia}
LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-auto}
LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https}
LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}
SERVER_TUNNEL_REMOTE_PORT=${SERVER_TUNNEL_REMOTE_PORT:-50051}
SSH_OPTS=${SSH_OPTS:-"-o BatchMode=yes -o ConnectTimeout=5"}
LESAVKA_CLIENT_RCT_MODE=${LESAVKA_CLIENT_RCT_MODE:-auto}
LESAVKA_CLIENT_RCT_UPSTREAM_MODE=${LESAVKA_CLIENT_RCT_UPSTREAM_MODE:-${LESAVKA_CLIENT_RCT_MODE}}
LESAVKA_CLIENT_RCT_UPSTREAM_CAMERA_CODEC=${LESAVKA_CLIENT_RCT_UPSTREAM_CAMERA_CODEC:-auto}
LESAVKA_CLIENT_RCT_UPSTREAM_AUDIO_CODEC=${LESAVKA_CLIENT_RCT_UPSTREAM_AUDIO_CODEC:-${LESAVKA_UPLINK_AUDIO_CODEC:-pcm}}
REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse}
REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}
REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-cfr}
REMOTE_PULSE_AUDIO_ANCHOR_SILENCE=${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE:-1}
REMOTE_CAPTURE_READY_TIMEOUT_SECONDS=${REMOTE_CAPTURE_READY_TIMEOUT_SECONDS:-30}
REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS:-1}
REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS:-3}
REMOTE_CAPTURE_PREROLL_SETTLE_SECONDS=${REMOTE_CAPTURE_PREROLL_SETTLE_SECONDS:-1}
LESAVKA_CLIENT_RCT_START_DELAY_SECONDS=${LESAVKA_CLIENT_RCT_START_DELAY_SECONDS:-0}
PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-20}
PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4}
PROBE_PULSE_PERIOD_MS=${PROBE_PULSE_PERIOD_MS:-1000}
PROBE_PULSE_WIDTH_MS=${PROBE_PULSE_WIDTH_MS:-120}
PROBE_MARKER_TICK_PERIOD=${PROBE_MARKER_TICK_PERIOD:-5}
PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}
PROBE_START_GRACE_SECONDS=${PROBE_START_GRACE_SECONDS:-12}
TAIL_SECONDS=${TAIL_SECONDS:-4}
PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_START_GRACE_SECONDS + TAIL_SECONDS + 20))}
CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_START_GRACE_SECONDS + TAIL_SECONDS))}
LESAVKA_CLIENT_RCT_MAX_AGE_MS=${LESAVKA_CLIENT_RCT_MAX_AGE_MS:-1000}
LESAVKA_CLIENT_RCT_MIN_PAIRS=${LESAVKA_CLIENT_RCT_MIN_PAIRS:-13}
LESAVKA_CLIENT_RCT_REQUIRE_SMOOTHNESS=${LESAVKA_CLIENT_RCT_REQUIRE_SMOOTHNESS:-0}
LESAVKA_CLIENT_RCT_SYNC_SAMPLE_INTERVAL_SECONDS=${LESAVKA_CLIENT_RCT_SYNC_SAMPLE_INTERVAL_SECONDS:-0.5}
LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}
STAMP="$(date +%Y%m%d-%H%M%S)"
LOCAL_REPORT_DIR="${LOCAL_OUTPUT_DIR%/}/lesavka-client-rct-transport-probe-${STAMP}"
LOCAL_CAPTURE="${LOCAL_REPORT_DIR}/capture.mkv"
LOCAL_CAPTURE_LOG="${LOCAL_REPORT_DIR}/capture.log"
LOCAL_REPORT_JSON="${LOCAL_REPORT_DIR}/report.json"
LOCAL_REPORT_TXT="${LOCAL_REPORT_DIR}/report.txt"
LOCAL_EVENTS_CSV="${LOCAL_REPORT_DIR}/events.csv"
LOCAL_CLIENT_TIMELINE_JSON="${LOCAL_REPORT_DIR}/client-transport-timeline.json"
LOCAL_CLOCK_ALIGNMENT_JSON="${LOCAL_REPORT_DIR}/clock-alignment.json"
LOCAL_TRANSPORT_SUMMARY_JSON="${LOCAL_REPORT_DIR}/client-rct-transport-summary.json"
LOCAL_TRANSPORT_SUMMARY_TXT="${LOCAL_REPORT_DIR}/client-rct-transport-summary.txt"
LOCAL_UPSTREAM_SYNC_JSONL="${LOCAL_REPORT_DIR}/upstream-sync-samples.jsonl"
LOCAL_UPSTREAM_SYNC_TXT="${LOCAL_REPORT_DIR}/upstream-sync-samples.txt"
LOCAL_CLIENT_SEND_JSONL="${LOCAL_REPORT_DIR}/client-send-bundles.jsonl"
LOCAL_UVC_FRAME_META_JSONL="${LOCAL_REPORT_DIR}/uvc-frame-meta.jsonl"
LOCAL_UVC_FRAME_META_SUMMARY_JSON="${LOCAL_REPORT_DIR}/uvc-frame-meta-summary.json"
LOCAL_UVC_FRAME_META_SUMMARY_TXT="${LOCAL_REPORT_DIR}/uvc-frame-meta-summary.txt"
LOCAL_RUN_LOG="${LOCAL_REPORT_DIR}/client-rct-run.log"
REMOTE_CAPTURE=${REMOTE_CAPTURE:-"/tmp/lesavka-client-rct-transport-probe-${STAMP}.mkv"}
LESAVKA_CLIENT_RCT_UVC_FRAME_META_LOG_REMOTE=${LESAVKA_CLIENT_RCT_UVC_FRAME_META_LOG_REMOTE:-}
LESAVKA_CLIENT_RCT_UVC_FRAME_META_LOG_REQUIRED=${LESAVKA_CLIENT_RCT_UVC_FRAME_META_LOG_REQUIRED:-0}
CAPTURE_READY_MARKER="__LESAVKA_CAPTURE_READY__"
mkdir -p "${LOCAL_REPORT_DIR}"
exec > >(tee -a "${LOCAL_RUN_LOG}") 2>&1
SERVER_TUNNEL_PID=""
CAPTURE_PID=""
RESOLVED_LESAVKA_SERVER_ADDR=""
cleanup() {
if [[ -n "${SERVER_TUNNEL_PID}" ]] && kill -0 "${SERVER_TUNNEL_PID}" >/dev/null 2>&1; then
kill "${SERVER_TUNNEL_PID}" >/dev/null 2>&1 || true
wait "${SERVER_TUNNEL_PID}" >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT
parse_mode() {
local mode=$1
if [[ "${mode}" == "auto" ]]; then
printf '0 0 0\n'
return 0
fi
if [[ "${mode}" =~ ^([0-9]+)x([0-9]+)@([0-9]+)$ ]]; then
printf '%s %s %s\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}"
return 0
fi
echo "invalid LESAVKA_CLIENT_RCT_MODE=${mode}; expected WIDTHxHEIGHT@FPS" >&2
return 1
}
pick_tunnel_port() {
python3 - <<'PY'
import socket
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
wait_for_local_port() {
local port=$1
local deadline=$(( $(date +%s) + 15 ))
until python3 - <<'PY' "${port}"
import socket
import sys
port = int(sys.argv[1])
try:
with socket.create_connection(("127.0.0.1", port), timeout=1):
pass
except OSError:
sys.exit(1)
PY
do
if (( $(date +%s) >= deadline )); then
echo "timed out waiting for localhost:${port}" >&2
return 1
fi
sleep 0.25
done
}
sleep_start_delay() {
[[ "${LESAVKA_CLIENT_RCT_START_DELAY_SECONDS}" =~ ^[0-9]+$ ]] || {
echo "LESAVKA_CLIENT_RCT_START_DELAY_SECONDS must be a non-negative number" >&2
exit 2
}
if (( LESAVKA_CLIENT_RCT_START_DELAY_SECONDS > 0 )); then
echo "==> delaying client-to-RCT transport probe start for ${LESAVKA_CLIENT_RCT_START_DELAY_SECONDS}s"
sleep "${LESAVKA_CLIENT_RCT_START_DELAY_SECONDS}"
fi
}
start_server_tunnel() {
if [[ "${LESAVKA_SERVER_ADDR}" != "auto" ]]; then
RESOLVED_LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_ADDR}"
return 0
fi
local local_port
local_port="$(pick_tunnel_port)"
echo "==> opening SSH tunnel to ${LESAVKA_SERVER_HOST}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT} on localhost:${local_port}"
ssh ${SSH_OPTS} -o ExitOnForwardFailure=yes \
-N -L "127.0.0.1:${local_port}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT}" \
"${LESAVKA_SERVER_HOST}" &
SERVER_TUNNEL_PID=$!
wait_for_local_port "${local_port}"
RESOLVED_LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_SCHEME}://127.0.0.1:${local_port}"
echo " ↪ tunneled to ${LESAVKA_SERVER_HOST}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT}"
}
sample_capture_clock_alignment() {
echo "==> sampling client/Tethys clock alignment for transport freshness"
python3 "${REPO_ROOT}/scripts/manual/client_rct_clock_alignment.py" \
"${TETHYS_HOST}" \
"${SSH_OPTS}" \
"${LOCAL_CLOCK_ALIGNMENT_JSON}"
}
build_probe_tools() {
echo "==> prebuilding client transport probe/analyzer"
(
cd "${REPO_ROOT}"
cargo build -p lesavka_client --bin lesavka-sync-probe --bin lesavka-sync-analyze --bin lesavka-relayctl
)
}
print_versions() {
echo "==> Lesavka versions under test"
(
cd "${REPO_ROOT}"
LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \
"${REPO_ROOT}/target/debug/lesavka-relayctl" \
--server "${RESOLVED_LESAVKA_SERVER_ADDR}" version
) | sed 's/^/ ↪ /' || true
}
start_tethys_capture() {
local width=$1 height=$2 fps=$3
echo "==> starting RCT UVC/UAC capture on ${TETHYS_HOST}"
ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
"${REMOTE_CAPTURE}" \
"${CAPTURE_SECONDS}" \
"${width}" \
"${height}" \
"${fps}" \
"${REMOTE_CAPTURE_STACK}" \
"${REMOTE_PULSE_CAPTURE_TOOL}" \
"${REMOTE_PULSE_VIDEO_MODE}" \
"${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE}" \
"${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS}" \
"${REMOTE_CAPTURE_PREROLL_SETTLE_SECONDS}" \
"${REMOTE_CAPTURE_READY_SETTLE_SECONDS}" \
"${CAPTURE_READY_MARKER}" \
>"${LOCAL_CAPTURE_LOG}" 2>&1 <"${SCRIPT_DIR}/client_rct_remote_capture.sh" &
CAPTURE_PID=$!
}
wait_for_capture_ready() {
local deadline=$(( $(date +%s) + REMOTE_CAPTURE_READY_TIMEOUT_SECONDS ))
until grep -q "${CAPTURE_READY_MARKER}" "${LOCAL_CAPTURE_LOG}" 2>/dev/null; do
if ! kill -0 "${CAPTURE_PID}" >/dev/null 2>&1; then
wait "${CAPTURE_PID}" || true
echo "RCT capture failed before the client probe could start; see ${LOCAL_CAPTURE_LOG}" >&2
exit 90
fi
if (( $(date +%s) >= deadline )); then
echo "timed out waiting for RCT capture readiness; see ${LOCAL_CAPTURE_LOG}" >&2
exit 90
fi
sleep 0.25
done
}
run_client_sync_probe() {
echo "==> running client-origin bundled transport probe against ${RESOLVED_LESAVKA_SERVER_ADDR}"
local camera_args=()
if [[ "${LESAVKA_CLIENT_RCT_UPSTREAM_MODE}" != "auto" ]]; then
camera_args+=(--camera-mode "${LESAVKA_CLIENT_RCT_UPSTREAM_MODE}")
fi
if [[ "${LESAVKA_CLIENT_RCT_UPSTREAM_CAMERA_CODEC}" != "auto" ]]; then
camera_args+=(--camera-codec "${LESAVKA_CLIENT_RCT_UPSTREAM_CAMERA_CODEC}")
fi
(
cd "${REPO_ROOT}"
LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \
LESAVKA_UPLINK_AUDIO_CODEC="${LESAVKA_CLIENT_RCT_UPSTREAM_AUDIO_CODEC}" \
LESAVKA_SYNC_PROBE_SEND_LOG="${LOCAL_CLIENT_SEND_JSONL}" \
timeout --signal=INT "${PROBE_TIMEOUT_SECONDS}" \
"${REPO_ROOT}/target/debug/lesavka-sync-probe" \
--server "${RESOLVED_LESAVKA_SERVER_ADDR}" \
--duration-seconds "${PROBE_DURATION_SECONDS}" \
--warmup-seconds "${PROBE_WARMUP_SECONDS}" \
--pulse-period-ms "${PROBE_PULSE_PERIOD_MS}" \
--pulse-width-ms "${PROBE_PULSE_WIDTH_MS}" \
--marker-tick-period "${PROBE_MARKER_TICK_PERIOD}" \
--event-width-codes "${PROBE_EVENT_WIDTH_CODES}" \
--timeline-json "${LOCAL_CLIENT_TIMELINE_JSON}" \
"${camera_args[@]}"
)
}
run_client_sync_probe_with_sampler() {
run_client_sync_probe &
local probe_pid=$!
python3 "${REPO_ROOT}/scripts/manual/client_rct_upstream_sync_sampler.py" \
"${REPO_ROOT}/target/debug/lesavka-relayctl" \
"${RESOLVED_LESAVKA_SERVER_ADDR}" \
"${LESAVKA_TLS_DOMAIN}" \
"${probe_pid}" \
"${LESAVKA_CLIENT_RCT_SYNC_SAMPLE_INTERVAL_SECONDS}" \
"${LOCAL_UPSTREAM_SYNC_JSONL}" \
"${LOCAL_UPSTREAM_SYNC_TXT}" &
local sampler_pid=$!
local probe_status=0
wait "${probe_pid}" || probe_status=$?
wait "${sampler_pid}" || true
return "${probe_status}"
}
fetch_and_analyze_capture() {
echo "==> fetching RCT capture back to ${LOCAL_CAPTURE}"
scp ${SSH_OPTS} "${TETHYS_HOST}:${REMOTE_CAPTURE}" "${LOCAL_CAPTURE}"
echo "==> analyzing RCT capture"
local analyze_args=("${LOCAL_CAPTURE}" --report-dir "${LOCAL_REPORT_DIR}")
if [[ -n "${PROBE_EVENT_WIDTH_CODES}" ]]; then
analyze_args+=(--event-width-codes "${PROBE_EVENT_WIDTH_CODES}")
fi
(
cd "${REPO_ROOT}"
"${REPO_ROOT}/target/debug/lesavka-sync-analyze" "${analyze_args[@]}"
)
}
write_transport_summary() {
python3 "${REPO_ROOT}/scripts/manual/client_rct_transport_summary.py" \
"${LOCAL_REPORT_JSON}" \
"${LOCAL_CLIENT_TIMELINE_JSON}" \
"${LOCAL_CAPTURE_LOG}" \
"${LOCAL_CLOCK_ALIGNMENT_JSON}" \
"${LOCAL_CAPTURE}" \
"${LOCAL_TRANSPORT_SUMMARY_JSON}" \
"${LOCAL_TRANSPORT_SUMMARY_TXT}" \
"${LESAVKA_CLIENT_RCT_MAX_AGE_MS}" \
"${LESAVKA_CLIENT_RCT_MIN_PAIRS}" \
"${LESAVKA_CLIENT_RCT_REQUIRE_SMOOTHNESS}"
}
fetch_and_summarize_uvc_frame_meta_log() {
SSH_OPTS="${SSH_OPTS}" "${REPO_ROOT}/scripts/manual/client_rct_uvc_frame_meta_fetch.sh" \
"${LESAVKA_SERVER_HOST}" \
"${LESAVKA_CLIENT_RCT_UVC_FRAME_META_LOG_REMOTE}" \
"${LESAVKA_CLIENT_RCT_UVC_FRAME_META_LOG_REQUIRED}" \
"${LOCAL_UVC_FRAME_META_JSONL}" \
"${LOCAL_UVC_FRAME_META_SUMMARY_JSON}" \
"${LOCAL_UVC_FRAME_META_SUMMARY_TXT}" \
"${LOCAL_CLIENT_TIMELINE_JSON}" \
"${MODE_FPS}" \
"${REPO_ROOT}"
}
read -r MODE_WIDTH MODE_HEIGHT MODE_FPS < <(parse_mode "${LESAVKA_CLIENT_RCT_MODE}")
echo "==> client-to-RCT bundled transport probe"
echo " ↪ mode=${LESAVKA_CLIENT_RCT_MODE}"
echo " ↪ upstream_mode=${LESAVKA_CLIENT_RCT_UPSTREAM_MODE} upstream_camera_codec=${LESAVKA_CLIENT_RCT_UPSTREAM_CAMERA_CODEC} upstream_audio_codec=${LESAVKA_CLIENT_RCT_UPSTREAM_AUDIO_CODEC}"
echo " ↪ capture_stack=${REMOTE_CAPTURE_STACK} pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL} video_mode=${REMOTE_PULSE_VIDEO_MODE}"
echo " ↪ server_addr=${LESAVKA_SERVER_ADDR}"
echo " ↪ max_client_to_rct_age_ms=${LESAVKA_CLIENT_RCT_MAX_AGE_MS}"
echo " ↪ start_delay=${LESAVKA_CLIENT_RCT_START_DELAY_SECONDS}s"
echo " ↪ uvc_frame_meta_log_remote=${LESAVKA_CLIENT_RCT_UVC_FRAME_META_LOG_REMOTE:-disabled}"
echo " ↪ artifact_dir=${LOCAL_REPORT_DIR}"
echo " ↪ run_log=${LOCAL_RUN_LOG}"
echo " ↪ no remote sudo/reconfigure will be attempted by this script"
sleep_start_delay
start_server_tunnel
build_probe_tools
print_versions
sample_capture_clock_alignment
start_tethys_capture "${MODE_WIDTH}" "${MODE_HEIGHT}" "${MODE_FPS}"
wait_for_capture_ready
sleep 1
run_client_sync_probe_with_sampler
capture_status=0
wait "${CAPTURE_PID}" || capture_status=$?
if [[ "${capture_status}" -ne 0 && "${capture_status}" -ne 124 ]]; then
echo "RCT capture exited with status ${capture_status}; see ${LOCAL_CAPTURE_LOG}" >&2
exit "${capture_status}"
fi
fetch_and_analyze_capture
write_transport_summary
fetch_and_summarize_uvc_frame_meta_log
echo "==> done"
printf '%s\n' \
"artifact_dir: ${LOCAL_REPORT_DIR}" "capture: ${LOCAL_CAPTURE}" \
"report_json: ${LOCAL_REPORT_JSON}" "report_txt: ${LOCAL_REPORT_TXT}" \
"events_csv: ${LOCAL_EVENTS_CSV}" "client_timeline_json: ${LOCAL_CLIENT_TIMELINE_JSON}" \
"clock_alignment_json: ${LOCAL_CLOCK_ALIGNMENT_JSON}" "transport_summary_json: ${LOCAL_TRANSPORT_SUMMARY_JSON}" \
"transport_summary_txt: ${LOCAL_TRANSPORT_SUMMARY_TXT}" "upstream_sync_jsonl: ${LOCAL_UPSTREAM_SYNC_JSONL}" \
"upstream_sync_txt: ${LOCAL_UPSTREAM_SYNC_TXT}" "client_send_jsonl: ${LOCAL_CLIENT_SEND_JSONL}" \
"uvc_frame_meta_jsonl: ${LOCAL_UVC_FRAME_META_JSONL}" "uvc_frame_meta_summary_json: ${LOCAL_UVC_FRAME_META_SUMMARY_JSON}" \
"uvc_frame_meta_summary_txt: ${LOCAL_UVC_FRAME_META_SUMMARY_TXT}" "run_log: ${LOCAL_RUN_LOG}"