lesavka/scripts/manual/run_upstream_browser_av_sync.sh

#!/usr/bin/env bash
# scripts/manual/run_upstream_browser_av_sync.sh
# Manual: browser consumer A/V sync hardware probe; not part of CI.
#
# Drive a real browser consumer on Tethys, record the combined MediaStream,
# pull the capture back, and analyze it with the Lesavka sync analyzer.

set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"

TETHYS_HOST=${TETHYS_HOST:-tethys}
LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-https://38.28.125.112:50051}
PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-15}
BROWSER_RECORD_SECONDS=${BROWSER_RECORD_SECONDS:-${PROBE_DURATION_SECONDS}}
BROWSER_SYNC_DRIVER_COMMAND=${BROWSER_SYNC_DRIVER_COMMAND:-}
BROWSER_CONSUMER_REUSE_SESSION=${BROWSER_CONSUMER_REUSE_SESSION:-0}
BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED:-1}
SYNC_ANALYZE_EVENT_WIDTH_CODES=${SYNC_ANALYZE_EVENT_WIDTH_CODES:-}
BROWSER_PORT=${BROWSER_PORT:-18443}
REMOTE_SCRIPT=${REMOTE_SCRIPT:-/tmp/lesavka-browser-consumer-probe.py}
REMOTE_CAPTURE=${REMOTE_CAPTURE:-/tmp/lesavka-browser-av-sync.webm}
REMOTE_STATUS=${REMOTE_STATUS:-/tmp/lesavka-browser-av-sync-status.json}
REMOTE_PROFILE_DIR=${REMOTE_PROFILE_DIR:-/tmp/lesavka-browser-probe-profile}
LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-"${REPO_ROOT}/tmp"}
SSH_OPTS=${SSH_OPTS:-"-o BatchMode=yes -o ConnectTimeout=5"}
DISPLAY_ENV=${DISPLAY_ENV:-":0"}
REMOTE_RUNTIME_DIR=${REMOTE_RUNTIME_DIR:-/run/user/1000}
REMOTE_DBUS_ADDRESS=${REMOTE_DBUS_ADDRESS:-}
REMOTE_XAUTHORITY=${REMOTE_XAUTHORITY:-}
READY_TIMEOUT_SECONDS=${READY_TIMEOUT_SECONDS:-120}

mkdir -p "${LOCAL_OUTPUT_DIR}"
STAMP="$(date +%Y%m%d-%H%M%S)"
LOCAL_CAPTURE="${LOCAL_OUTPUT_DIR}/lesavka-browser-av-sync-${STAMP}.webm"
LOCAL_REPORT_DIR="${LOCAL_OUTPUT_DIR}/lesavka-browser-av-sync-${STAMP}"

scp ${SSH_OPTS} "${REPO_ROOT}/scripts/manual/browser_consumer_probe.py" "${TETHYS_HOST}:${REMOTE_SCRIPT}"

ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
  "${REMOTE_SCRIPT}" \
  "${REMOTE_CAPTURE}" \
  "${REMOTE_STATUS}" \
  "${REMOTE_PROFILE_DIR}" \
  "${BROWSER_RECORD_SECONDS}" \
  "${BROWSER_PORT}" \
  "${DISPLAY_ENV}" \
  "${REMOTE_RUNTIME_DIR}" \
  "${BROWSER_CONSUMER_REUSE_SESSION}" <<'REMOTE_SETUP'
set -euo pipefail
remote_script=$1
remote_capture=$2
remote_status=$3
remote_profile_dir=$4
duration=$5
port=$6
display_env=$7
runtime_dir=$8
reuse_session=$9
dbus_address=""
xauthority_path=""
if [[ "${reuse_session}" == "1" ]] && status_json="$(curl --max-time 2 -fsS "http://127.0.0.1:${port}/status" 2>/dev/null)"; then
  if STATUS_JSON="${status_json}" python3 - <<'PY'
import json
import os
import sys

status = json.loads(os.environ["STATUS_JSON"])
sys.exit(0 if status.get("ready") else 1)
PY
  then
    exit 0
  fi
fi
firefox_pid="$(pgrep -n -x firefox-esr || true)"
if [[ -n "${firefox_pid}" && -r "/proc/${firefox_pid}/environ" ]]; then
  while IFS='=' read -r key value; do
    case "$key" in
      DBUS_SESSION_BUS_ADDRESS) dbus_address="$value" ;;
      XAUTHORITY) xauthority_path="$value" ;;
      DISPLAY) [[ -z "${display_env}" || "${display_env}" == ":0" ]] && display_env="$value" ;;
    esac
  done < <(tr '\0' '\n' <"/proc/${firefox_pid}/environ")
fi
[[ -z "${dbus_address}" ]] && dbus_address="unix:path=${runtime_dir}/bus"
fuser -k "${port}/tcp" >/dev/null 2>&1 || true
pkill -f "firefox.*${remote_profile_dir}" >/dev/null 2>&1 || true
for _ in $(seq 1 20); do
  if ! pgrep -f "firefox.*${remote_profile_dir}" >/dev/null 2>&1; then
    break
  fi
  sleep 0.25
done
rm -f "$remote_capture" "$remote_status"
rm -rf "$remote_profile_dir"
mkdir -p "$remote_profile_dir"
cat >"${remote_profile_dir}/user.js" <<'FIREFOX_PREFS'
user_pref("media.navigator.permission.disabled", true);
user_pref("permissions.default.camera", 1);
user_pref("permissions.default.microphone", 1);
user_pref("media.autoplay.default", 0);
user_pref("media.autoplay.blocking_policy", 0);
user_pref("toolkit.telemetry.reportingpolicy.firstRun", false);
user_pref("browser.shell.checkDefaultBrowser", false);
user_pref("browser.tabs.warnOnClose", false);
user_pref("browser.startup.page", 1);
user_pref("browser.startup.homepage_override.mstone", "ignore");
user_pref("startup.homepage_welcome_url", "");
user_pref("startup.homepage_welcome_url.additional", "");
user_pref("browser.aboutwelcome.enabled", false);
user_pref("trailhead.firstrun.didSeeAboutWelcome", true);
FIREFOX_PREFS
printf 'user_pref("browser.startup.homepage", "http://127.0.0.1:%s/");\n' "$port" >>"${remote_profile_dir}/user.js"
nohup python3 "$remote_script" --port "$port" --output "$remote_capture" --status "$remote_status" --duration-seconds "$duration" >/tmp/lesavka-browser-consumer-probe.log 2>&1 &
if [[ -n "${xauthority_path}" ]]; then
  nohup env DISPLAY="$display_env" XDG_RUNTIME_DIR="$runtime_dir" DBUS_SESSION_BUS_ADDRESS="$dbus_address" XAUTHORITY="$xauthority_path" \
    firefox --new-instance --no-remote --profile "$remote_profile_dir" \
    >/tmp/lesavka-browser-consumer-firefox.log 2>&1 &
else
  nohup env DISPLAY="$display_env" XDG_RUNTIME_DIR="$runtime_dir" DBUS_SESSION_BUS_ADDRESS="$dbus_address" \
    firefox --new-instance --no-remote --profile "$remote_profile_dir" \
    >/tmp/lesavka-browser-consumer-firefox.log 2>&1 &
fi
REMOTE_SETUP

echo "==> waiting for browser consumer to become ready on ${TETHYS_HOST}"
deadline=$(( $(date +%s) + READY_TIMEOUT_SECONDS ))
while true; do
  status_json=$(ssh ${SSH_OPTS} "${TETHYS_HOST}" "test -f '${REMOTE_STATUS}' && cat '${REMOTE_STATUS}'" || true)
  if [[ -n "${status_json}" ]]; then
    if STATUS_JSON="${status_json}" python3 -c 'import json, os, sys; status = json.loads(os.environ["STATUS_JSON"]); sys.exit(0 if status.get("ready") else 1)'
    then
      echo "==> browser consumer ready"
      break
    fi
  fi
  if (( $(date +%s) >= deadline )); then
    echo "browser consumer did not become ready before timeout" >&2
    [[ -n "${status_json:-}" ]] && echo "last status: ${status_json}" >&2
    exit 1
  fi
  sleep 1

done

echo "==> triggering browser recording"
start_json=""
if ! start_json="$(ssh ${SSH_OPTS} "${TETHYS_HOST}" "curl --max-time 10 -fsS -X POST http://127.0.0.1:${BROWSER_PORT}/start")"; then
  status_json=$(ssh ${SSH_OPTS} "${TETHYS_HOST}" "test -f '${REMOTE_STATUS}' && cat '${REMOTE_STATUS}'" || true)
  echo "browser consumer start request failed or timed out" >&2
  [[ -n "${status_json:-}" ]] && echo "last status: ${status_json}" >&2
  exit 1
fi
browser_start_token="$(START_JSON="${start_json}" python3 - <<'PY'
import json
import os

status = json.loads(os.environ["START_JSON"])
print(int(status.get("start_token") or 0))
PY
)"
echo "   ↪ browser_start_token=${browser_start_token}"

sleep 1

if [[ -n "${BROWSER_SYNC_DRIVER_COMMAND}" ]]; then
  echo "==> running custom browser sync driver"
  bash -lc "${BROWSER_SYNC_DRIVER_COMMAND}"
else
  echo "==> running local Lesavka sync probe against ${LESAVKA_SERVER_ADDR}"
  (
    cd "${REPO_ROOT}"
    cargo run -p lesavka_client --bin lesavka-sync-probe -- \
      --server "${LESAVKA_SERVER_ADDR}" \
      --duration-seconds "${PROBE_DURATION_SECONDS}"
  )
fi

echo "==> waiting for browser recording upload"
deadline_upload=$(( $(date +%s) + PROBE_DURATION_SECONDS + 60 ))
while true; do
  status_json=$(ssh ${SSH_OPTS} "${TETHYS_HOST}" "test -f '${REMOTE_STATUS}' && cat '${REMOTE_STATUS}'" || true)
  if [[ -n "${status_json}" ]]; then
    if STATUS_JSON="${status_json}" BROWSER_START_TOKEN="${browser_start_token}" python3 - <<'PY'
import json
import os
import sys

status = json.loads(os.environ["STATUS_JSON"])
expected = int(os.environ["BROWSER_START_TOKEN"])
uploaded = bool(status.get("uploaded"))
uploaded_token = status.get("uploaded_start_token")
sys.exit(0 if uploaded and uploaded_token == expected else 1)
PY
    then
      echo "==> browser recording uploaded"
      break
    fi
  fi
  if (( $(date +%s) >= deadline_upload )); then
    echo "browser recording was not uploaded before timeout" >&2
    [[ -n "${status_json:-}" ]] && echo "last status: ${status_json}" >&2
    exit 1
  fi
  sleep 1

done

echo "==> fetching capture back to ${LOCAL_CAPTURE}"
fetch_status=1
for attempt in 1 2 3 4 5; do
  if scp ${SSH_OPTS} "${TETHYS_HOST}:${REMOTE_CAPTURE}" "${LOCAL_CAPTURE}"; then
    fetch_status=0
    break
  fi
  echo "capture fetch attempt ${attempt} failed; retrying" >&2
  sleep 3
done
if ((fetch_status != 0)); then
  echo "failed to fetch browser capture from ${TETHYS_HOST}:${REMOTE_CAPTURE}" >&2
  exit "${fetch_status}"
fi

echo "==> analyzing browser capture"
mkdir -p "${LOCAL_REPORT_DIR}"
analyze_args=(--report-dir "${LOCAL_REPORT_DIR}")
if [[ -n "${SYNC_ANALYZE_EVENT_WIDTH_CODES}" ]]; then
  analyze_args+=(--event-width-codes "${SYNC_ANALYZE_EVENT_WIDTH_CODES}")
fi
analyze_args+=("${LOCAL_CAPTURE}")
analysis_log="${LOCAL_REPORT_DIR}/analyze.log"
analysis_status=0
(
  cd "${REPO_ROOT}"
  cargo run -p lesavka_client --bin lesavka-sync-analyze -- \
    "${analyze_args[@]}"
) 2>&1 | tee "${analysis_log}" || analysis_status=$?
if (( analysis_status != 0 )); then
  python3 - "${analysis_log}" "${analysis_status}" "${LOCAL_REPORT_DIR}/analysis-failure.json" <<'PY'
import json
import re
import sys
from pathlib import Path

log_path = Path(sys.argv[1])
status = int(sys.argv[2])
output_path = Path(sys.argv[3])
text = log_path.read_text(encoding="utf-8", errors="replace")
reason = ""
lines = text.splitlines()
for index, line in enumerate(lines):
    if "Caused by:" in line:
        for candidate in lines[index + 1:]:
            candidate = candidate.strip()
            if candidate:
                reason = candidate
                break
if not reason:
    reason = lines[-1].strip() if lines else "analyzer failed"

raw_match = re.search(
    r"raw activity delta was ([+-]?[0-9]+(?:\\.[0-9]+)?) ms "
    r"\\(video=([0-9]+(?:\\.[0-9]+)?)s audio=([0-9]+(?:\\.[0-9]+)?)s\\)",
    text,
)
paired_match = re.search(r"saw ([0-9]+)", reason)
payload = {
    "status": "analysis_error",
    "exit_status": status,
    "reason": reason,
    "analyze_log": str(log_path),
    "paired_pulses": int(paired_match.group(1)) if paired_match else 0,
}
if raw_match:
    payload.update({
        "raw_activity_delta_ms": float(raw_match.group(1)),
        "raw_first_video_activity_s": float(raw_match.group(2)),
        "raw_first_audio_activity_s": float(raw_match.group(3)),
    })
output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
PY
  echo "==> analyzer failed for ${LOCAL_CAPTURE}"
  echo "   ↪ analysis_failure_json=${LOCAL_REPORT_DIR}/analysis-failure.json"
  if [[ "${BROWSER_ANALYSIS_REQUIRED}" == "1" ]]; then
    exit "${analysis_status}"
  fi
  echo "   ↪ continuing because BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED}"
fi

echo "==> done"
echo "capture: ${LOCAL_CAPTURE}"
echo "report_dir: ${LOCAL_REPORT_DIR}"