lesavka/scripts/manual/run_server_to_rc_mode_matrix.sh

#!/usr/bin/env bash
# scripts/manual/run_server_to_rc_mode_matrix.sh
# Manual: validate server-generated UVC/UAC output against the RC target across
# the UVC modes the UI advertises. This is still a hardware-in-the-loop probe:
# it captures the real Tethys UVC/UAC endpoints and summarizes sync,
# freshness, and smoothness for each mode.
#
# Reconfigure mode is intentionally a fast runtime path: it updates the remote
# Lesavka env files and cycles the UVC gadget, but it does not rebuild or
# reinstall the server binary for each mode.

set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)"

if (( EUID == 0 )); then
  printf 'Do not run this matrix with local sudo.\n' >&2
  printf 'Run it as your normal workstation user; the script will request sudo on the remote server host only when reconfiguring UVC.\n' >&2
  exit 64
fi

TETHYS_HOST=${TETHYS_HOST:-tethys}
LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST:-theia}
LESAVKA_SERVER_CONNECT_HOST=${LESAVKA_SERVER_CONNECT_HOST:-38.28.125.112}
LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-auto}
LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https}
LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}
LESAVKA_SERVER_REPO=${LESAVKA_SERVER_REPO:-auto}
SSH_OPTS=${SSH_OPTS:-"-o BatchMode=yes -o ConnectTimeout=30"}

LESAVKA_SERVER_RC_CORE_WEBCAM_MODES=${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES:-1280x720@20,1280x720@30,1920x1080@20,1920x1080@30}
LESAVKA_SERVER_RC_MODES=${LESAVKA_SERVER_RC_MODES:-${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES}}
LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US:-${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}}
LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US:-170000}
LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US:-1280x720@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1280x720@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US}}
LESAVKA_SERVER_RC_MODE_DELAYS_US=${LESAVKA_SERVER_RC_MODE_DELAYS_US:-1280x720@20=170000,1280x720@30=170000,1920x1080@20=170000,1920x1080@30=170000}
LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES=${LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES:-1280x720,1920x1080}
LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS=${LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS:-20,30}
LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX:-Logitech|BRIO|C9[0-9]+|HD UVC WebCam|USB2[.]0 HD|Integrated Camera|Webcam|Camera}
LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX:-Lesavka|UGREEN|MACROSILICON|Composite|Capture}
LESAVKA_SERVER_RC_MODE_SOURCE=${LESAVKA_SERVER_RC_MODE_SOURCE:-configured}
LESAVKA_SERVER_RC_RECONFIGURE=${LESAVKA_SERVER_RC_RECONFIGURE:-0}
LESAVKA_SERVER_RC_RECONFIGURE_REF=${LESAVKA_SERVER_RC_RECONFIGURE_REF:-master}
LESAVKA_SERVER_RC_RECONFIGURE_UPDATE=${LESAVKA_SERVER_RC_RECONFIGURE_UPDATE:-0}
LESAVKA_SERVER_RC_RECONFIGURE_COMMAND=${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND:-}
LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY:-runtime}
LESAVKA_SERVER_RC_RECONFIGURE_CODEC=${LESAVKA_SERVER_RC_RECONFIGURE_CODEC:-mjpeg}
LESAVKA_SERVER_RC_ALLOW_GADGET_RESET=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET:-1}
LESAVKA_SERVER_RC_FORCE_GADGET_REBUILD=${LESAVKA_SERVER_RC_FORCE_GADGET_REBUILD:-1}
LESAVKA_SERVER_RC_RECONFIGURE_SETTLE_SECONDS=${LESAVKA_SERVER_RC_RECONFIGURE_SETTLE_SECONDS:-4}
LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE=${LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE:-0}
LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY=${LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY:-1}
LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD=${LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD:-}
LESAVKA_SERVER_RC_WAIT_TETHYS_READY=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY:-1}
LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS:-60}
LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS:-6}
LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS:-3}
LESAVKA_SERVER_RC_PROBE_PREBUILD=${LESAVKA_SERVER_RC_PROBE_PREBUILD:-1}
LESAVKA_SERVER_RC_CONTINUE_ON_FAIL=${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL:-1}
LESAVKA_SERVER_RC_TUNE_DELAYS=${LESAVKA_SERVER_RC_TUNE_DELAYS:-1}
LESAVKA_SERVER_RC_TUNE_CONFIRM=${LESAVKA_SERVER_RC_TUNE_CONFIRM:-1}
LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-8}
LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS:-1000}
LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS:-80}
LESAVKA_SERVER_RC_TUNE_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US:-500000}
LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000}

LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350}
LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS:-100}
LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}
LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS:-1}
LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS=${LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS:-1}

LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS=${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS:-0}
LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS=${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS:-0}
LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS=${LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS:-1}
LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS=${LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS:-1}
LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES:-12}
LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES:-12}
LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES:-5}
LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS=${LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS:-2}

PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-20}
PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4}
PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,1,3,2,4,1,1,3,1,4,2,1,2,3,4,1,3,2,2,4,1,2,4,3,1,1,4,2,3,1,2}
REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}
REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-cfr}
REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse}
REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto}
REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}
LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}

STAMP="$(date +%Y%m%d-%H%M%S)"
LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}
MATRIX_REPORT_DIR=${MATRIX_REPORT_DIR:-"${LOCAL_OUTPUT_DIR%/}/lesavka-server-rc-mode-matrix-${STAMP}"}
MATRIX_SUMMARY_JSON="${MATRIX_REPORT_DIR}/mode-matrix-summary.json"
MATRIX_SUMMARY_CSV="${MATRIX_REPORT_DIR}/mode-matrix-summary.csv"
MATRIX_SUMMARY_TXT="${MATRIX_REPORT_DIR}/mode-matrix-summary.txt"
MATRIX_DELAY_JSON="${MATRIX_REPORT_DIR}/mode-delay-recommendations.json"
MATRIX_DELAY_ENV="${MATRIX_REPORT_DIR}/mode-delay-recommendations.env"
mkdir -p "${MATRIX_REPORT_DIR}"

mode_id() {
  local mode=$1
  printf '%s\n' "${mode//@/_}" | tr -c '[:alnum:]_.-' '_'
}

parse_mode() {
  local mode=$1
  if [[ ! "${mode}" =~ ^([0-9]+)x([0-9]+)@([0-9]+)$ ]]; then
    printf 'invalid mode %s; expected WIDTHxHEIGHT@FPS\n' "${mode}" >&2
    exit 64
  fi
  printf '%s %s %s\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}"
}

lookup_mode_delay_us() {
  local mode=$1
  local delay_map=$2
  local default_value=$3
  local entry key value
  IFS=',' read -r -a delay_entries <<<"${delay_map}"
  for entry in "${delay_entries[@]}"; do
    key=${entry%%=*}
    value=${entry#*=}
    if [[ "${key}" == "${mode}" && "${value}" =~ ^-?[0-9]+$ ]]; then
      printf '%s\n' "${value}"
      return 0
    fi
  done
  printf '%s\n' "${default_value}"
}

lookup_audio_delay_us() {
  lookup_mode_delay_us "$1" "${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}" "${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US}"
}

lookup_video_delay_us() {
  lookup_mode_delay_us "$1" "${LESAVKA_SERVER_RC_MODE_DELAYS_US}" "${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US}"
}

artifact_dir_from_log() {
  local log_path=$1
  local search_dir=$2
  local artifact_dir
  artifact_dir="$(awk -F': ' '/^artifact_dir: / {print $2}' "${log_path}" 2>/dev/null | tail -n1 || true)"
  if [[ -z "${artifact_dir}" ]]; then
    artifact_dir="$(find "${search_dir}" -mindepth 1 -maxdepth 1 -type d -name 'lesavka-output-delay-probe-*' | sort | tail -n1 || true)"
  fi
  printf '%s\n' "${artifact_dir}"
}

RUN_MODE_PROBE_STATUS=0
run_mode_probe() {
  local width=$1
  local height=$2
  local fps=$3
  local audio_delay_us=$4
  local video_delay_us=$5
  local output_dir=$6
  local log_path=$7

  set +e
  TETHYS_HOST="${TETHYS_HOST}" \
  LESAVKA_SERVER_HOST="${LESAVKA_SERVER_HOST}" \
  LESAVKA_SERVER_CONNECT_HOST="${LESAVKA_SERVER_CONNECT_HOST}" \
  LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_ADDR}" \
  LESAVKA_SERVER_SCHEME="${LESAVKA_SERVER_SCHEME}" \
  LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \
  SSH_OPTS="${SSH_OPTS}" \
  REMOTE_PULSE_CAPTURE_TOOL="${REMOTE_PULSE_CAPTURE_TOOL}" \
  REMOTE_PULSE_VIDEO_MODE="${REMOTE_PULSE_VIDEO_MODE}" \
  REMOTE_CAPTURE_STACK="${REMOTE_CAPTURE_STACK}" \
  REMOTE_AUDIO_SOURCE="${REMOTE_AUDIO_SOURCE}" \
  REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK="${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" \
  REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS="${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}" \
  PROBE_PREBUILD=0 \
  VIDEO_SIZE="${width}x${height}" \
  VIDEO_FPS="${fps}" \
  VIDEO_FORMAT=mjpeg \
  REMOTE_EXPECT_UVC_WIDTH="${width}" \
  REMOTE_EXPECT_UVC_HEIGHT="${height}" \
  REMOTE_EXPECT_UVC_FPS="${fps}" \
  LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US="${audio_delay_us}" \
  LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US="${video_delay_us}" \
  LESAVKA_OUTPUT_DELAY_APPLY=0 \
  LESAVKA_OUTPUT_DELAY_SAVE=0 \
  LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0 \
  LESAVKA_OUTPUT_DELAY_MIN_PAIRS="${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}" \
  LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS="${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}" \
  LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" \
  LESAVKA_OUTPUT_DELAY_MAX_STEP_US="${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}" \
  LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" \
  LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}" \
  LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}" \
  PROBE_EVENT_WIDTH_CODES="${PROBE_EVENT_WIDTH_CODES}" \
  PROBE_DURATION_SECONDS="${PROBE_DURATION_SECONDS}" \
  PROBE_WARMUP_SECONDS="${PROBE_WARMUP_SECONDS}" \
  LOCAL_OUTPUT_DIR="${output_dir}" \
    "${SCRIPT_DIR}/run_upstream_av_sync.sh" 2>&1 | tee "${log_path}"
  RUN_MODE_PROBE_STATUS=${PIPESTATUS[0]}
  set -e
}

write_tune_candidate_env() {
  local mode_result=$1
  local output_env=$2
  python3 - <<'PY' \
    "${mode_result}" \
    "${output_env}" \
    "${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}" \
    "${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}" \
    "${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" \
    "${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}" \
    "${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}"
import json
import math
import pathlib
import shlex
import sys

(
    result_path,
    output_env_path,
    min_pairs_raw,
    max_abs_skew_raw,
    max_drift_raw,
    max_step_raw,
    min_change_raw,
) = sys.argv[1:8]


def env_line(key, value):
    return f"{key}={shlex.quote(str(value))}\n"


def as_int(value, default=0):
    try:
        return int(str(value).strip())
    except Exception:
        return default


def as_float(value, default=0.0):
    try:
        result = float(str(value).strip())
    except Exception:
        return default
    return result if math.isfinite(result) else default


result = json.loads(pathlib.Path(result_path).read_text())
calibration = result.get("output_delay_calibration") or {}
sync = result.get("sync") or {}
min_pairs = max(1, as_int(min_pairs_raw, 8))
max_abs_skew_ms = max(1.0, as_float(max_abs_skew_raw, 1000.0))
max_drift_ms = max(0.0, as_float(max_drift_raw, 80.0))
max_step_us = max(1, as_int(max_step_raw, 500_000))
min_change_us = max(0, as_int(min_change_raw, 5_000))
current_audio = as_int(result.get("audio_delay_us"), 0)
current_video = as_int(result.get("video_delay_us"), 0)
target_audio = as_int(calibration.get("audio_target_offset_us"), current_audio)
target_video = as_int(calibration.get("video_target_offset_us"), current_video)
paired = as_int(calibration.get("paired_event_count"), as_int(sync.get("paired_event_count"), 0))
drift_ms = as_float(calibration.get("drift_ms"), as_float(sync.get("drift_ms"), 0.0))
max_abs_skew_ms_observed = as_float(
    calibration.get("max_abs_skew_ms"),
    as_float(sync.get("p95_abs_skew_ms"), 0.0),
)
delta_audio = target_audio - current_audio
delta_video = target_video - current_video
raw_delta_us = as_int(
    calibration.get("raw_device_delta_us"),
    delta_video if abs(delta_video) >= abs(delta_audio) else -delta_audio,
)
reasons = []

if result.get("run_status") != 0:
    reasons.append(f"seed probe exited {result.get('run_status')}")
if paired < min_pairs:
    reasons.append(f"paired_event_count {paired} < {min_pairs}")
if max_abs_skew_ms_observed > max_abs_skew_ms:
    reasons.append(
        f"max_abs_skew_ms {max_abs_skew_ms_observed:.1f} > {max_abs_skew_ms:.1f}"
    )
if abs(drift_ms) > max_drift_ms:
    reasons.append(f"abs(drift_ms) {abs(drift_ms):.1f} > {max_drift_ms:.1f}")
if abs(raw_delta_us) > max_step_us:
    reasons.append(f"raw delay correction {raw_delta_us:+d}us exceeds {max_step_us}us")
if target_audio < 0 or target_video < 0:
    reasons.append(
        f"direct confirmation delays must be non-negative, got audio={target_audio} video={target_video}"
    )
if abs(delta_audio) < min_change_us and abs(delta_video) < min_change_us:
    reasons.append(
        f"target change audio={delta_audio:+d}us video={delta_video:+d}us is below {min_change_us}us"
    )

ready = not reasons
values = {
    "tune_ready": str(ready).lower(),
    "tune_reason": "ready" if ready else "; ".join(reasons),
    "tune_audio_delay_us": target_audio,
    "tune_video_delay_us": target_video,
    "tune_audio_delta_us": delta_audio,
    "tune_video_delta_us": delta_video,
    "tune_paired_event_count": paired,
    "tune_drift_ms": f"{drift_ms:.3f}",
    "tune_max_abs_skew_ms": f"{max_abs_skew_ms_observed:.3f}",
    "tune_raw_delta_us": raw_delta_us,
}
with pathlib.Path(output_env_path).open("w") as handle:
    for key, value in values.items():
        handle.write(env_line(key, value))
PY
}

discover_local_webcam_modes() {
  if ! command -v python3 >/dev/null 2>&1; then
    printf 'LESAVKA_SERVER_RC_MODES=auto requires python3 for local webcam mode discovery.\n' >&2
    exit 64
  fi
  if ! command -v v4l2-ctl >/dev/null 2>&1; then
    printf 'LESAVKA_SERVER_RC_MODES=auto requires v4l2-ctl for local webcam mode discovery.\n' >&2
    exit 64
  fi
  python3 - <<'PY' \
    "${LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES}" \
    "${LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS}" \
    "${LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX}" \
    "${LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX}"
import glob
import re
import subprocess
import sys

sizes_raw, fps_raw, include_raw, exclude_raw = sys.argv[1:5]

def parse_sizes(raw):
    sizes = set()
    for entry in raw.split(","):
        entry = entry.strip()
        if not entry:
            continue
        match = re.fullmatch(r"(\d+)x(\d+)", entry)
        if not match:
            raise SystemExit(f"invalid discovery size {entry!r}; expected WIDTHxHEIGHT")
        sizes.add((int(match.group(1)), int(match.group(2))))
    return sizes

def parse_fps(raw):
    values = set()
    for entry in raw.split(","):
        entry = entry.strip()
        if not entry:
            continue
        if not entry.isdigit():
            raise SystemExit(f"invalid discovery fps {entry!r}; expected integer FPS")
        values.add(int(entry))
    return values

allowed_sizes = parse_sizes(sizes_raw)
allowed_fps = parse_fps(fps_raw)
include = re.compile(include_raw, re.IGNORECASE) if include_raw else None
exclude = re.compile(exclude_raw, re.IGNORECASE) if exclude_raw else None
all_modes = set()
camera_modes = []

for dev in sorted(glob.glob("/dev/video*")):
    try:
        info = subprocess.run(
            ["v4l2-ctl", "-d", dev, "--info"],
            check=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.DEVNULL,
            text=True,
        ).stdout
        formats = subprocess.run(
            ["v4l2-ctl", "-d", dev, "--list-formats-ext"],
            check=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.DEVNULL,
            text=True,
        ).stdout
    except Exception:
        continue

    card = dev
    for line in info.splitlines():
        if "Card type" in line:
            card = line.split(":", 1)[1].strip()
            break
    label = f"{card} {dev}"
    if exclude and exclude.search(label):
        continue
    if include and not include.search(label):
        continue

    current_size = None
    modes = set()
    for line in formats.splitlines():
        size_match = re.search(r"Size:\s+Discrete\s+(\d+)x(\d+)", line)
        if size_match:
            current_size = (int(size_match.group(1)), int(size_match.group(2)))
            continue
        fps_match = re.search(r"Interval:\s+Discrete\s+[^()]+\((\d+(?:\.\d+)?) fps\)", line)
        if not fps_match or current_size not in allowed_sizes:
            continue
        fps_float = float(fps_match.group(1))
        fps = int(round(fps_float))
        if abs(fps_float - fps) > 0.05 or fps not in allowed_fps:
            continue
        modes.add((current_size[0], current_size[1], fps))

    if modes:
        all_modes.update(modes)
        camera_modes.append((label, modes))

if not all_modes:
    print(
        "no matching local webcam modes found; "
        f"sizes={sizes_raw} fps={fps_raw} include={include_raw!r} exclude={exclude_raw!r}",
        file=sys.stderr,
    )
    raise SystemExit(64)

for label, modes in camera_modes:
    rendered = ",".join(f"{w}x{h}@{fps}" for w, h, fps in sorted(modes))
    print(f"   -> local webcam {label}: {rendered}", file=sys.stderr)

print(",".join(f"{w}x{h}@{fps}" for w, h, fps in sorted(all_modes)), end="")
PY
}

clear_remote_sudo_password() {
  LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD=""
}
trap clear_remote_sudo_password EXIT

ensure_remote_sudo_password() {
  [[ "${LESAVKA_SERVER_RC_RECONFIGURE}" != "0" ]] || return 0
  [[ -z "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}" ]] || return 0
  [[ -n "${LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD}" ]] && return 0

  if [[ "${LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY}" == "0" ]]; then
    printf 'remote sudo password is required for automatic reconfigure; set LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD or leave LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY=1.\n' >&2
    exit 64
  fi
  if [[ ! -t 0 ]]; then
    printf 'remote sudo password is required, but stdin is not a terminal.\n' >&2
    printf 'Re-run from a terminal or set LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD in the environment.\n' >&2
    exit 64
  fi

  printf 'Theia sudo password for %s: ' "${LESAVKA_SERVER_HOST}" >&2
  IFS= read -r -s LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD
  printf '\n' >&2
}

run_remote_root_script() {
  local description=$1
  shift
  local script_file remote_wrapper ssh_cmd status
  script_file="$(mktemp)"
  cat >"${script_file}"

  remote_wrapper='set -euo pipefail
read -r __lesavka_sudo_password
__lesavka_script=$(mktemp)
cleanup() { rm -f "$__lesavka_script"; }
trap cleanup EXIT
cat >"$__lesavka_script"
printf "%s\n" "$__lesavka_sudo_password" | sudo -S -p "" -v
printf "%s\n" "$__lesavka_sudo_password" | sudo -S -p "" bash "$__lesavka_script" "$@"
'
  printf -v ssh_cmd 'bash -c %q _' "${remote_wrapper}"
  for arg in "$@"; do
    printf -v ssh_cmd '%s %q' "${ssh_cmd}" "${arg}"
  done

  set +e
  {
    printf '%s\n' "${LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD}"
    cat "${script_file}"
  } | ssh ${SSH_OPTS} "${LESAVKA_SERVER_HOST}" "${ssh_cmd}"
  status=$?
  set -e
  rm -f "${script_file}"
  if [[ "${status}" -ne 0 ]]; then
    printf '%s failed on %s with exit %s\n' "${description}" "${LESAVKA_SERVER_HOST}" "${status}" >&2
  fi
  return "${status}"
}

prime_remote_sudo() {
  [[ "${LESAVKA_SERVER_RC_RECONFIGURE}" != "0" ]] || return 0
  [[ -z "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}" ]] || return 0
  ensure_remote_sudo_password
  echo "==> priming remote sudo on ${LESAVKA_SERVER_HOST}"
  run_remote_root_script "remote sudo prime" <<'REMOTE_SUDO_PRIME'
set -euo pipefail
true
REMOTE_SUDO_PRIME
}

prebuild_probe_tools() {
  [[ "${LESAVKA_SERVER_RC_PROBE_PREBUILD}" != "0" ]] || return 0
  echo "==> prebuilding relay control/analyzer once for the mode matrix"
  (
    cd "${REPO_ROOT}"
    cargo build -p lesavka_client --bin lesavka-sync-analyze --bin lesavka-relayctl
  )
}

reconfigure_server_mode() {
  local mode=$1
  local width=$2
  local height=$3
  local fps=$4
  [[ "${LESAVKA_SERVER_RC_RECONFIGURE}" != "0" ]] || return 0

  echo "==> reconfiguring ${LESAVKA_SERVER_HOST} UVC gadget for ${mode}"
  if [[ -n "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}" ]]; then
    LESAVKA_MODE="${mode}" \
    LESAVKA_UVC_WIDTH="${width}" \
    LESAVKA_UVC_HEIGHT="${height}" \
    LESAVKA_UVC_FPS="${fps}" \
      bash -c "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}"
    return 0
  fi

  if [[ "${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY}" != "runtime" ]]; then
    printf 'unsupported LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY=%s\n' "${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY}" >&2
    printf 'Use runtime for fast mode changes, or LESAVKA_SERVER_RC_RECONFIGURE_COMMAND for a custom install/reconfigure command.\n' >&2
    return 64
  fi

  local interval
  interval=$((10000000 / fps))
  run_remote_root_script "runtime UVC reconfigure for ${mode}" \
    "${mode}" \
    "${width}" \
    "${height}" \
    "${fps}" \
    "${interval}" \
    "${LESAVKA_SERVER_RC_RECONFIGURE_CODEC}" \
    "${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET}" \
    "${LESAVKA_SERVER_RC_FORCE_GADGET_REBUILD}" \
    "${LESAVKA_SERVER_RC_RECONFIGURE_SETTLE_SECONDS}" \
    "${LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE}" <<'REMOTE_RECONFIGURE'
set -euo pipefail
mode=$1
width=$2
height=$3
fps=$4
interval=$5
codec=$6
allow_gadget_reset=$7
force_gadget_rebuild=$8
settle_seconds=$9
verbose=${10}

set_env_value() {
  local file=$1
  local key=$2
  local value=$3
  local tmp
  tmp=$(mktemp)
  if [[ -f "${file}" ]]; then
    awk -v key="${key}" -v value="${value}" '
      BEGIN { wrote = 0 }
      $0 ~ "^" key "=" {
        print key "=" value
        wrote = 1
        next
      }
      { print }
      END {
        if (!wrote) {
          print key "=" value
        }
      }
    ' "${file}" >"${tmp}"
  else
    printf '%s=%s\n' "${key}" "${value}" >"${tmp}"
  fi
  install -m 0644 "${tmp}" "${file}"
  rm -f "${tmp}"
}

if [[ ! -x /usr/local/bin/lesavka-core.sh ]]; then
  printf 'missing /usr/local/bin/lesavka-core.sh; run the server installer once before using fast runtime reconfigure.\n' >&2
  exit 65
fi
if [[ ! -x /usr/local/bin/lesavka-server || ! -x /usr/local/bin/lesavka-uvc ]]; then
  printf 'missing installed Lesavka binaries; run the server installer once before using fast runtime reconfigure.\n' >&2
  exit 65
fi

install -d -m 0755 /etc/lesavka
touch /etc/lesavka/server.env /etc/lesavka/uvc.env

set_env_value /etc/lesavka/server.env LESAVKA_CAM_OUTPUT uvc
set_env_value /etc/lesavka/server.env LESAVKA_UVC_CODEC "${codec}"
set_env_value /etc/lesavka/server.env LESAVKA_UVC_WIDTH "${width}"
set_env_value /etc/lesavka/server.env LESAVKA_UVC_HEIGHT "${height}"
set_env_value /etc/lesavka/server.env LESAVKA_UVC_FPS "${fps}"
set_env_value /etc/lesavka/server.env LESAVKA_UVC_INTERVAL "${interval}"

set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_CODEC "${codec}"
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_WIDTH "${width}"
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_HEIGHT "${height}"
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_FPS "${fps}"
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_INTERVAL "${interval}"

printf '   ↪ fast runtime env updated: CAM_OUTPUT=uvc UVC_MODE=%s codec=%s\n' "${mode}" "${codec}"
systemctl daemon-reload
systemctl stop lesavka-server >/dev/null 2>&1 || true
systemctl stop lesavka-uvc >/dev/null 2>&1 || true
systemctl reset-failed lesavka-core lesavka-uvc lesavka-server >/dev/null 2>&1 || true

if [[ "${allow_gadget_reset}" != "0" && "${force_gadget_rebuild}" != "0" ]]; then
  printf '   ↪ cycling UVC gadget descriptors for %s\n' "${mode}"
  mode_log_id=$(printf '%s' "${mode}" | tr -c '[:alnum:]_.-' '_')
  core_log="/tmp/lesavka-core-reconfigure-${mode_log_id}.log"
  core_cmd=(
    env
    LESAVKA_ALLOW_GADGET_RESET=1 \
    LESAVKA_FORCE_GADGET_REBUILD=1 \
    LESAVKA_ATTACH_WRITE_UDC=1 \
    LESAVKA_DETACH_CLEAR_UDC=1 \
    LESAVKA_UVC_FALLBACK=0 \
    LESAVKA_UVC_CODEC="${codec}" \
    LESAVKA_UVC_WIDTH="${width}" \
    LESAVKA_UVC_HEIGHT="${height}" \
    LESAVKA_UVC_FPS="${fps}" \
    LESAVKA_UVC_INTERVAL="${interval}" \
    /usr/local/bin/lesavka-core.sh
  )
  if [[ "${verbose}" != "0" ]]; then
    "${core_cmd[@]}"
  else
    if "${core_cmd[@]}" >"${core_log}" 2>&1; then
      printf '   ↪ lesavka-core reconfigure log: %s\n' "${core_log}"
    else
      rc=$?
      printf 'lesavka-core reconfigure failed; tail of %s follows\n' "${core_log}" >&2
      tail -n 80 "${core_log}" >&2 || true
      exit "${rc}"
    fi
  fi
elif [[ "${allow_gadget_reset}" != "0" ]]; then
  printf '   ↪ gadget reset allowed, but force rebuild disabled; refreshing services only\n'
else
  printf '   ↪ preserving attached UVC gadget; descriptors may remain on the previous mode\n'
fi

systemctl start lesavka-uvc
systemctl restart lesavka-server
sleep "${settle_seconds}"
systemctl is-active lesavka-core lesavka-uvc lesavka-server >/dev/null
printf '   ↪ services active after %s reconfigure\n' "${mode}"
REMOTE_RECONFIGURE
}

wait_tethys_media_ready() {
  local mode=$1
  local width=$2
  local height=$3
  local fps=$4
  [[ "${LESAVKA_SERVER_RC_WAIT_TETHYS_READY}" != "0" ]] || return 0

  echo "==> waiting for Tethys media endpoints for ${mode}"
  ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
    "${mode}" \
    "${width}" \
    "${height}" \
    "${fps}" \
    "${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}" \
    "${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}" \
    "${REMOTE_CAPTURE_STACK}" \
    "${REMOTE_AUDIO_SOURCE}" <<'REMOTE_TETHYS_READY'
set -euo pipefail
mode=$1
width=$2
height=$3
fps=$4
timeout_seconds=$5
settle_seconds=$6
capture_stack=$7
audio_source=$8

sleep "${settle_seconds}"

find_lesavka_video_device() {
  if [[ -d /dev/v4l/by-id ]]; then
    while IFS= read -r path; do
      [[ -e "${path}" ]] || continue
      printf '%s\n' "${path}"
      return 0
    done < <(find /dev/v4l/by-id -maxdepth 1 -type l -name '*Lesavka*video-index0' 2>/dev/null | sort)
  fi

  if command -v v4l2-ctl >/dev/null 2>&1; then
    v4l2-ctl --list-devices 2>/dev/null \
      | awk '
          BEGIN { want=0 }
          /Lesavka Composite|Lesavka.*UVC/ { want=1; next }
          /^[^ \t]/ { want=0 }
          want && /^[ \t]+\/dev\/video[0-9]+/ {
            gsub(/^[ \t]+/, "", $0)
            print
            exit
          }
        '
  fi
}

video_ready() {
  local dev=$1
  [[ -n "${dev}" ]] || return 1
  [[ -e "${dev}" ]] || return 1
  if ! command -v v4l2-ctl >/dev/null 2>&1; then
    return 0
  fi

  local listing
  listing="$(v4l2-ctl -d "${dev}" --list-formats-ext 2>/dev/null || true)"
  grep -q "Size: Discrete ${width}x${height}" <<<"${listing}" || return 1
  grep -Eq "(^|[^0-9])${fps}(\\.0+)? fps" <<<"${listing}" || return 1
}

pulse_ready() {
  if ! command -v pactl >/dev/null 2>&1; then
    return 1
  fi
  if [[ "${audio_source}" == pulse:* ]]; then
    local requested=${audio_source#pulse:}
    pactl list short sources 2>/dev/null | awk -v requested="${requested}" '$2 == requested { found=1 } END { exit found ? 0 : 1 }'
    return
  fi
  pactl list short sources 2>/dev/null | grep -Eq 'alsa_input\..*Lesavka_Composite|Lesavka_Composite'
}

alsa_ready() {
  if [[ "${audio_source}" == alsa:* ]]; then
    [[ -e "${audio_source#alsa:}" ]] || return 1
    return 0
  fi
  command -v arecord >/dev/null 2>&1 || return 1
  arecord -l 2>/dev/null | grep -Eq 'Lesavka|UAC2_Gadget|UAC2Gadget|Composite'
}

audio_ready() {
  case "${capture_stack}" in
    pulse)
      pulse_ready
      ;;
    alsa)
      alsa_ready
      ;;
    auto)
      pulse_ready || alsa_ready
      ;;
    pwpipe)
      command -v pw-dump >/dev/null 2>&1 && pw-dump 2>/dev/null | grep -Eq 'Lesavka_Composite|Lesavka Composite'
      ;;
    *)
      pulse_ready || alsa_ready
      ;;
  esac
}

deadline=$((SECONDS + timeout_seconds))
last_video='none'
last_audio='none'
while (( SECONDS <= deadline )); do
  video_dev="$(find_lesavka_video_device || true)"
  last_video="${video_dev:-none}"
  if [[ -n "${video_dev}" ]] && video_ready "${video_dev}"; then
    if audio_ready; then
      printf '   ↪ Tethys media ready: video=%s mode=%s audio_stack=%s\n' "${video_dev}" "${mode}" "${capture_stack}"
      exit 0
    fi
    last_audio='not-ready'
  fi
  sleep 0.5
done

printf 'timed out waiting for Tethys Lesavka media endpoints for %s after %ss\n' "${mode}" "${timeout_seconds}" >&2
printf 'last video candidate: %s\n' "${last_video}" >&2
printf 'last audio candidate: %s\n' "${last_audio}" >&2
if command -v v4l2-ctl >/dev/null 2>&1; then
  v4l2-ctl --list-devices >&2 || true
  if [[ "${last_video}" != "none" ]]; then
    v4l2-ctl -d "${last_video}" --list-formats-ext >&2 || true
  fi
fi
if command -v pactl >/dev/null 2>&1; then
  pactl list short sources | grep -iE 'lesavka|uac|composite' >&2 || true
fi
if command -v arecord >/dev/null 2>&1; then
  arecord -l >&2 || true
fi
exit 70
REMOTE_TETHYS_READY
}

write_mode_result() {
  local mode=$1
  local width=$2
  local height=$3
  local fps=$4
  local video_delay_us=$5
  local audio_delay_us=$6
  local run_status=$7
  local run_log=$8
  local artifact_dir=$9
  local output_json=${10}

  python3 - <<'PY' \
    "${mode}" \
    "${width}" \
    "${height}" \
    "${fps}" \
    "${video_delay_us}" \
    "${audio_delay_us}" \
    "${run_status}" \
    "${run_log}" \
    "${artifact_dir}" \
    "${output_json}" \
    "${LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS}" \
    "${LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS}" \
    "${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS}" \
    "${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS}" \
    "${LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS}" \
    "${LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS}" \
    "${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES}" \
    "${LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES}" \
    "${LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES}" \
    "${LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS}"
import json
import math
import pathlib
import sys

(
    mode,
    width_raw,
    height_raw,
    fps_raw,
    video_delay_raw,
    audio_delay_raw,
    run_status_raw,
    run_log,
    artifact_dir_raw,
    output_json,
    require_sync_raw,
    require_freshness_raw,
    max_video_hiccups_raw,
    max_audio_hiccups_raw,
    max_video_jitter_raw,
    max_audio_jitter_raw,
    max_missing_raw,
    max_undecodable_raw,
    max_duplicates_raw,
    max_low_rms_raw,
) = sys.argv[1:]


def as_bool(value):
    return str(value).strip().lower() not in {"", "0", "false", "no", "off"}


def as_int(value, default=0):
    try:
        return int(str(value).strip())
    except Exception:
        return default


def as_float(value, default=0.0):
    try:
        result = float(str(value).strip())
    except Exception:
        return default
    return result if math.isfinite(result) else default


def load_json(path):
    try:
        return json.loads(pathlib.Path(path).read_text())
    except Exception:
        return {}


def nested(mapping, *keys, default=None):
    value = mapping
    for key in keys:
        if not isinstance(value, dict):
            return default
        value = value.get(key)
    return default if value is None else value


artifact_dir = pathlib.Path(artifact_dir_raw) if artifact_dir_raw else pathlib.Path()
report = load_json(artifact_dir / "report.json")
correlation = load_json(artifact_dir / "output-delay-correlation.json")
calibration = load_json(artifact_dir / "output-delay-calibration.json")
freshness = correlation.get("freshness") or {}
smoothness = correlation.get("smoothness") or {}
video = smoothness.get("video") or {}
audio = smoothness.get("audio") or {}
audio_cadence = audio.get("packet_cadence") or {}
audio_rms = audio.get("rms_continuity") or {}
verdict = report.get("verdict") or {}

sync_pass = verdict.get("passed") is True
freshness_status = freshness.get("status", "unknown")
freshness_pass = freshness_status == "pass"
run_status = as_int(run_status_raw, 1)

video_hiccups = as_int(video.get("hiccup_count"), 0)
audio_hiccups = as_int(audio_cadence.get("hiccup_count"), 0)
video_jitter = as_float(nested(video, "jitter_stats", "p95_jitter_ms"), 0.0)
audio_jitter = as_float(nested(audio_cadence, "jitter_stats", "p95_jitter_ms"), 0.0)
missing = as_int(video.get("estimated_missing_frames"), 0)
undecodable = as_int(video.get("undecodable_frames"), 0)
duplicates = as_int(video.get("duplicate_frames"), 0)
low_rms = as_int(audio_rms.get("low_rms_window_count"), 0)
pair_confidences = []
for event in report.get("paired_events") or []:
    if not isinstance(event, dict):
        continue
    try:
        confidence = float(event.get("confidence"))
    except Exception:
        continue
    if math.isfinite(confidence):
        pair_confidences.append(confidence)


def median(values, default=0.0):
    if not values:
        return default
    ordered = sorted(values)
    mid = len(ordered) // 2
    if len(ordered) % 2:
        return ordered[mid]
    return (ordered[mid - 1] + ordered[mid]) / 2.0


activity_start_delta_ms = as_float(report.get("activity_start_delta_ms"), 0.0)
first_skew_ms = as_float(report.get("first_skew_ms"), 0.0)
activity_pair_disagreement_ms = (
    activity_start_delta_ms - first_skew_ms
    if as_int(report.get("paired_event_count"), 0) > 0
    else 0.0
)

reasons = []
if run_status != 0:
    reasons.append(f"probe command exited {run_status}")
if as_bool(require_sync_raw) and not sync_pass:
    reasons.append(f"sync did not pass: {verdict.get('status', 'unknown')}")
if as_bool(require_freshness_raw) and not freshness_pass:
    reasons.append(f"freshness did not pass: {freshness_status}")
if video_hiccups > as_int(max_video_hiccups_raw):
    reasons.append(f"video hiccups {video_hiccups} > {max_video_hiccups_raw}")
if audio_hiccups > as_int(max_audio_hiccups_raw):
    reasons.append(f"audio hiccups {audio_hiccups} > {max_audio_hiccups_raw}")
if video_jitter > as_float(max_video_jitter_raw):
    reasons.append(f"video p95 jitter {video_jitter:.1f}ms > {as_float(max_video_jitter_raw):.1f}ms")
if audio_jitter > as_float(max_audio_jitter_raw):
    reasons.append(f"audio p95 jitter {audio_jitter:.1f}ms > {as_float(max_audio_jitter_raw):.1f}ms")
if missing > as_int(max_missing_raw):
    reasons.append(f"estimated missing video frames {missing} > {max_missing_raw}")
if undecodable > as_int(max_undecodable_raw):
    reasons.append(f"undecodable video frames {undecodable} > {max_undecodable_raw}")
if duplicates > as_int(max_duplicates_raw):
    reasons.append(f"duplicate video frames {duplicates} > {max_duplicates_raw}")
if low_rms > as_int(max_low_rms_raw):
    reasons.append(f"low-RMS audio windows {low_rms} > {max_low_rms_raw}")

artifact = {
    "schema": "lesavka.server-rc-mode-result.v1",
    "mode": mode,
    "width": as_int(width_raw),
    "height": as_int(height_raw),
    "fps": as_int(fps_raw),
    "audio_delay_us": as_int(audio_delay_raw),
    "video_delay_us": as_int(video_delay_raw),
    "run_status": run_status,
    "run_log": run_log,
    "artifact_dir": str(artifact_dir),
    "report_json": str(artifact_dir / "report.json"),
    "correlation_json": str(artifact_dir / "output-delay-correlation.json"),
    "calibration_json": str(artifact_dir / "output-delay-calibration.json"),
    "passed": not reasons,
    "failure_reasons": reasons,
    "sync": {
        "passed": sync_pass,
        "status": verdict.get("status", "unknown"),
        "reason": verdict.get("reason", ""),
        "p95_abs_skew_ms": as_float(verdict.get("p95_abs_skew_ms"), 0.0),
        "median_skew_ms": as_float(report.get("median_skew_ms"), 0.0),
        "drift_ms": as_float(report.get("drift_ms"), 0.0),
        "activity_start_delta_ms": activity_start_delta_ms,
        "first_skew_ms": first_skew_ms,
        "activity_pair_disagreement_ms": activity_pair_disagreement_ms,
        "video_event_count": as_int(report.get("video_event_count"), 0),
        "audio_event_count": as_int(report.get("audio_event_count"), 0),
        "paired_event_count": as_int(report.get("paired_event_count"), 0),
        "paired_confidence_min": min(pair_confidences) if pair_confidences else 0.0,
        "paired_confidence_median": median(pair_confidences),
    },
    "freshness": {
        "status": freshness_status,
        "reason": freshness.get("reason", ""),
        "worst_event_age_p95_ms": freshness.get("worst_event_age_p95_ms"),
        "clock_uncertainty_ms": freshness.get("clock_uncertainty_ms"),
        "worst_event_age_with_uncertainty_ms": freshness.get("worst_event_age_with_uncertainty_ms"),
        "worst_freshness_drift_ms": freshness.get("worst_freshness_drift_ms"),
        "max_age_limit_ms": freshness.get("max_age_limit_ms"),
    },
    "smoothness": {
        "video_frames": as_int(video.get("timestamps"), 0),
        "video_p95_jitter_ms": video_jitter,
        "video_max_interval_ms": as_float(nested(video, "interval_stats", "max_interval_ms"), 0.0),
        "video_hiccups": video_hiccups,
        "video_decoded_frames": as_int(video.get("decoded_frames"), 0),
        "video_duplicate_frames": duplicates,
        "video_estimated_missing_frames": missing,
        "video_undecodable_frames": undecodable,
        "audio_packets": as_int(audio_cadence.get("timestamps"), 0),
        "audio_p95_jitter_ms": audio_jitter,
        "audio_max_interval_ms": as_float(nested(audio_cadence, "interval_stats", "max_interval_ms"), 0.0),
        "audio_hiccups": audio_hiccups,
        "audio_low_rms_windows": low_rms,
        "audio_median_rms": as_float(nested(audio_rms, "rms_stats", "median_rms"), 0.0),
    },
    "output_delay_calibration": {
        "ready": calibration.get("ready") is True,
        "decision": calibration.get("decision", "unknown"),
        "target": calibration.get("target", ""),
        "paired_event_count": as_int(calibration.get("paired_event_count"), 0),
        "min_pairs": as_int(calibration.get("min_pairs"), 0),
        "measured_device_skew_ms": as_float(calibration.get("measured_device_skew_ms"), 0.0),
        "p95_abs_skew_ms": as_float(calibration.get("p95_abs_skew_ms"), 0.0),
        "max_abs_skew_ms": as_float(calibration.get("max_abs_skew_ms"), 0.0),
        "drift_ms": as_float(calibration.get("drift_ms"), 0.0),
        "active_audio_offset_us": as_int(calibration.get("active_audio_offset_us"), as_int(audio_delay_raw)),
        "active_video_offset_us": as_int(calibration.get("active_video_offset_us"), as_int(video_delay_raw)),
        "audio_offset_adjust_us": as_int(calibration.get("audio_offset_adjust_us"), 0),
        "video_offset_adjust_us": as_int(calibration.get("video_offset_adjust_us"), 0),
        "audio_target_offset_us": as_int(calibration.get("audio_target_offset_us"), 0),
        "video_target_offset_us": as_int(calibration.get("video_target_offset_us"), 0),
        "raw_device_delta_us": as_int(calibration.get("raw_device_delta_us"), 0),
        "bounded_device_delta_us": as_int(calibration.get("bounded_device_delta_us"), 0),
        "max_step_us": as_int(calibration.get("max_step_us"), 0),
        "note": calibration.get("note", ""),
    },
}
pathlib.Path(output_json).write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n")
PY
}

summarize_matrix() {
  python3 - <<'PY' "${MATRIX_REPORT_DIR}" "${MATRIX_SUMMARY_JSON}" "${MATRIX_SUMMARY_CSV}" "${MATRIX_SUMMARY_TXT}" "${MATRIX_DELAY_JSON}" "${MATRIX_DELAY_ENV}"
import csv
import json
import pathlib
import shlex
import sys

root = pathlib.Path(sys.argv[1])
summary_json = pathlib.Path(sys.argv[2])
summary_csv = pathlib.Path(sys.argv[3])
summary_txt = pathlib.Path(sys.argv[4])
delay_json = pathlib.Path(sys.argv[5])
delay_env = pathlib.Path(sys.argv[6])
results = []
for path in sorted(root.glob("*/mode-result.json")):
    try:
        result = json.loads(path.read_text())
        seed_path = path.with_name("mode-result-seed.json")
        tuned_path = path.with_name("mode-result-tuned.json")
        if seed_path.exists():
            result["seed_result_json"] = str(seed_path)
            try:
                seed = json.loads(seed_path.read_text())
                result["seed_audio_delay_us"] = seed.get("audio_delay_us")
                result["seed_video_delay_us"] = seed.get("video_delay_us")
            except Exception:
                pass
        if tuned_path.exists():
            result["tuned_result_json"] = str(tuned_path)
        results.append(result)
    except Exception:
        continue

delay_recommendations = {}
video_delay_entries = []
audio_delay_entries = []
for result in results:
    mode = result.get("mode")
    if not mode:
        continue
    sync = result.get("sync") or {}
    calibration = result.get("output_delay_calibration") or {}
    required_pairs = calibration.get("min_pairs") or 8
    confirmed = (
        sync.get("passed") is True
        and (sync.get("paired_event_count") or 0) >= required_pairs
    )
    candidate_video = calibration.get("video_target_offset_us")
    candidate_audio = calibration.get("audio_target_offset_us")
    video_delay = result.get("video_delay_us")
    audio_delay = result.get("audio_delay_us")
    status = "confirmed" if confirmed else "tested"
    candidate_available = (
        calibration.get("ready") is True
        and (calibration.get("paired_event_count") or 0) > 0
        and isinstance(candidate_video, int)
        and isinstance(candidate_audio, int)
    )
    if not confirmed and candidate_available:
        video_delay = candidate_video
        audio_delay = candidate_audio
        status = "candidate_unconfirmed"
    elif not confirmed:
        status = "unavailable"
        video_delay = None
        audio_delay = None
    if status != "unavailable" and isinstance(video_delay, int) and isinstance(audio_delay, int):
        video_delay_entries.append(f"{mode}={video_delay}")
        audio_delay_entries.append(f"{mode}={audio_delay}")
    delay_recommendations[mode] = {
        "status": status,
        "audio_delay_us": audio_delay,
        "video_delay_us": video_delay,
        "tested_audio_delay_us": result.get("audio_delay_us"),
        "tested_video_delay_us": result.get("video_delay_us"),
        "sync_status": sync.get("status"),
        "median_skew_ms": sync.get("median_skew_ms"),
        "paired_event_count": sync.get("paired_event_count"),
        "paired_confidence_median": sync.get("paired_confidence_median"),
        "activity_pair_disagreement_ms": sync.get("activity_pair_disagreement_ms"),
    }

summary = {
    "schema": "lesavka.server-rc-mode-matrix-summary.v1",
    "artifact_dir": str(root),
    "passed": bool(results) and all(result.get("passed") for result in results),
    "mode_count": len(results),
    "delay_recommendations_json": str(delay_json),
    "delay_recommendations_env": str(delay_env),
    "delay_recommendations": delay_recommendations,
    "results": results,
}
summary_json.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
delay_json.write_text(
    json.dumps(
        {
            "schema": "lesavka.server-rc-mode-delay-recommendations.v1",
            "artifact_dir": str(root),
            "video_delays_us": {
                mode: entry.get("video_delay_us")
                for mode, entry in delay_recommendations.items()
            },
            "audio_delays_us": {
                mode: entry.get("audio_delay_us")
                for mode, entry in delay_recommendations.items()
            },
            "recommendations": delay_recommendations,
        },
        indent=2,
        sort_keys=True,
    )
    + "\n"
)
delay_env.write_text(
    "LESAVKA_SERVER_RC_MODE_DELAYS_US="
    + shlex.quote(",".join(video_delay_entries))
    + "\n"
    + "LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US="
    + shlex.quote(",".join(audio_delay_entries))
    + "\n"
)

fieldnames = [
    "mode",
    "passed",
    "seed_video_delay_us",
    "seed_audio_delay_us",
    "video_delay_us",
    "audio_delay_us",
    "sync_status",
    "p95_abs_skew_ms",
    "median_skew_ms",
    "sync_drift_ms",
    "freshness_status",
    "freshness_budget_ms",
    "freshness_drift_ms",
    "video_hiccups",
    "video_missing",
    "video_undecodable",
    "audio_hiccups",
    "audio_low_rms",
    "calibration_ready",
    "calibration_decision",
    "calibration_target",
    "calibration_measured_skew_ms",
    "calibration_video_target_offset_us",
    "calibration_audio_target_offset_us",
    "artifact_dir",
]
with summary_csv.open("w", newline="", encoding="utf-8") as handle:
    writer = csv.DictWriter(handle, fieldnames=fieldnames)
    writer.writeheader()
    for result in results:
        writer.writerow({
            "mode": result.get("mode"),
            "passed": result.get("passed"),
            "seed_video_delay_us": result.get("seed_video_delay_us"),
            "seed_audio_delay_us": result.get("seed_audio_delay_us"),
            "video_delay_us": result.get("video_delay_us"),
            "audio_delay_us": result.get("audio_delay_us"),
            "sync_status": (result.get("sync") or {}).get("status"),
            "p95_abs_skew_ms": (result.get("sync") or {}).get("p95_abs_skew_ms"),
            "median_skew_ms": (result.get("sync") or {}).get("median_skew_ms"),
            "sync_drift_ms": (result.get("sync") or {}).get("drift_ms"),
            "freshness_status": (result.get("freshness") or {}).get("status"),
            "freshness_budget_ms": (result.get("freshness") or {}).get("worst_event_age_with_uncertainty_ms"),
            "freshness_drift_ms": (result.get("freshness") or {}).get("worst_freshness_drift_ms"),
            "video_hiccups": (result.get("smoothness") or {}).get("video_hiccups"),
            "video_missing": (result.get("smoothness") or {}).get("video_estimated_missing_frames"),
            "video_undecodable": (result.get("smoothness") or {}).get("video_undecodable_frames"),
            "audio_hiccups": (result.get("smoothness") or {}).get("audio_hiccups"),
            "audio_low_rms": (result.get("smoothness") or {}).get("audio_low_rms_windows"),
            "calibration_ready": (result.get("output_delay_calibration") or {}).get("ready"),
            "calibration_decision": (result.get("output_delay_calibration") or {}).get("decision"),
            "calibration_target": (result.get("output_delay_calibration") or {}).get("target"),
            "calibration_measured_skew_ms": (result.get("output_delay_calibration") or {}).get("measured_device_skew_ms"),
            "calibration_video_target_offset_us": (result.get("output_delay_calibration") or {}).get("video_target_offset_us"),
            "calibration_audio_target_offset_us": (result.get("output_delay_calibration") or {}).get("audio_target_offset_us"),
            "artifact_dir": result.get("artifact_dir"),
        })

lines = [
    f"Server-to-RC mode matrix for {root}",
    f"- modes: {len(results)}",
    f"- verdict: {'pass' if summary['passed'] else 'fail'}",
]
for result in results:
    sync = result.get("sync") or {}
    freshness = result.get("freshness") or {}
    smooth = result.get("smoothness") or {}
    calibration = result.get("output_delay_calibration") or {}
    marker = "PASS" if result.get("passed") else "FAIL"
    lines.append(
        f"- {marker} {result.get('mode')}: "
        f"delays video={result.get('video_delay_us', 0)}us audio={result.get('audio_delay_us', 0)}us; "
        f"sync {sync.get('status')} p95={sync.get('p95_abs_skew_ms', 0.0):.1f}ms median={sync.get('median_skew_ms', 0.0):+.1f}ms; "
        f"freshness {freshness.get('status')} budget={freshness.get('worst_event_age_with_uncertainty_ms') or 0.0:.1f}ms; "
        f"smooth video hiccups={smooth.get('video_hiccups', 0)} missing={smooth.get('video_estimated_missing_frames', 0)} undecodable={smooth.get('video_undecodable_frames', 0)} audio hiccups={smooth.get('audio_hiccups', 0)}"
    )
    lines.append(
        "  calibration: "
        f"{calibration.get('decision', 'unknown')} ready={calibration.get('ready', False)} "
        f"target={calibration.get('target', '')} measured={calibration.get('measured_device_skew_ms', 0.0):+.1f}ms "
        f"video_target={calibration.get('video_target_offset_us', 0)}us "
        f"audio_target={calibration.get('audio_target_offset_us', 0)}us"
    )
    lines.append(
        "  sync evidence: "
        f"video_onsets={sync.get('video_event_count', 0)} audio_onsets={sync.get('audio_event_count', 0)} "
        f"pairs={sync.get('paired_event_count', 0)} "
        f"pair_conf_median={sync.get('paired_confidence_median', 0.0):.3f} "
        f"raw_pair_disagreement={sync.get('activity_pair_disagreement_ms', 0.0):+.1f}ms"
    )
    if "seed_video_delay_us" in result or "seed_audio_delay_us" in result:
        lines.append(
            "  tuning: "
            f"seed video={result.get('seed_video_delay_us', 0)}us audio={result.get('seed_audio_delay_us', 0)}us -> "
            f"tested video={result.get('video_delay_us', 0)}us audio={result.get('audio_delay_us', 0)}us"
        )
    for reason in result.get("failure_reasons") or []:
        lines.append(f"  reason: {reason}")
if video_delay_entries or audio_delay_entries:
    lines.append(f"- recommended video delays: {','.join(video_delay_entries)}")
    lines.append(f"- recommended audio delays: {','.join(audio_delay_entries)}")
summary_txt.write_text("\n".join(lines) + "\n")
print("\n".join(lines))
PY
}

if [[ "${LESAVKA_SERVER_RC_MODES}" == "auto" ]]; then
  echo "==> discovering local webcam-backed Lesavka modes"
  LESAVKA_SERVER_RC_MODES="$(discover_local_webcam_modes)"
  LESAVKA_SERVER_RC_MODE_SOURCE=local-v4l2
fi

echo "==> server-to-RC mode matrix"
echo "   ↪ modes=${LESAVKA_SERVER_RC_MODES}"
echo "   ↪ mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}"
echo "   ↪ video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}"
echo "   ↪ audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}"
echo "   ↪ capture_stack=${REMOTE_CAPTURE_STACK} audio_source=${REMOTE_AUDIO_SOURCE} pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL} video_mode=${REMOTE_PULSE_VIDEO_MODE}"
echo "   ↪ tune_delays=${LESAVKA_SERVER_RC_TUNE_DELAYS} confirm=${LESAVKA_SERVER_RC_TUNE_CONFIRM} min_pairs=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS} max_abs_skew_ms=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS} max_step_us=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US} min_change_us=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}"
echo "   ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}"
echo "   ↪ reconfigure=${LESAVKA_SERVER_RC_RECONFIGURE} strategy=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY} allow_gadget_reset=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET}"
echo "   ↪ tethys_ready=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY} settle=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}s timeout=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}s preroll_discard=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}s"
echo "   ↪ artifact_dir=${MATRIX_REPORT_DIR}"

prime_remote_sudo
prebuild_probe_tools

IFS=',' read -r -a modes <<<"${LESAVKA_SERVER_RC_MODES}"
for mode in "${modes[@]}"; do
  mode="${mode//[[:space:]]/}"
  [[ -n "${mode}" ]] || continue
  read -r width height fps < <(parse_mode "${mode}")
  video_delay_us="$(lookup_video_delay_us "${mode}")"
  audio_delay_us="$(lookup_audio_delay_us "${mode}")"
  id="$(mode_id "${mode}")"
  mode_dir="${MATRIX_REPORT_DIR}/${id}"
  mode_log="${mode_dir}/mode-run.log"
  mode_result="${mode_dir}/mode-result.json"
  seed_result="${mode_dir}/mode-result-seed.json"
  tuned_log="${mode_dir}/mode-tuned-run.log"
  tuned_result="${mode_dir}/mode-result-tuned.json"
  tune_env="${mode_dir}/mode-tune-candidate.env"
  mkdir -p "${mode_dir}"

  echo "==> mode ${mode}: video_delay_us=${video_delay_us} audio_delay_us=${audio_delay_us}"
  reconfigure_server_mode "${mode}" "${width}" "${height}" "${fps}"
  wait_tethys_media_ready "${mode}" "${width}" "${height}" "${fps}"

  run_mode_probe "${width}" "${height}" "${fps}" "${audio_delay_us}" "${video_delay_us}" "${mode_dir}" "${mode_log}"
  run_status=${RUN_MODE_PROBE_STATUS}
  artifact_dir="$(artifact_dir_from_log "${mode_log}" "${mode_dir}")"
  write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${video_delay_us}" "${audio_delay_us}" "${run_status}" "${mode_log}" "${artifact_dir}" "${mode_result}"
  cp "${mode_result}" "${seed_result}"

  if [[ "${LESAVKA_SERVER_RC_TUNE_DELAYS}" != "0" && "${LESAVKA_SERVER_RC_TUNE_CONFIRM}" != "0" ]]; then
    write_tune_candidate_env "${seed_result}" "${tune_env}"
    # shellcheck disable=SC1090
    source "${tune_env}"
    if [[ "${tune_ready:-false}" == "true" ]]; then
      echo "==> mode ${mode}: confirming tuned delays video_delay_us=${tune_video_delay_us} audio_delay_us=${tune_audio_delay_us}"
      echo "   ↪ tune delta: video=${tune_video_delta_us}us audio=${tune_audio_delta_us}us pairs=${tune_paired_event_count} drift=${tune_drift_ms}ms"
      run_mode_probe "${width}" "${height}" "${fps}" "${tune_audio_delay_us}" "${tune_video_delay_us}" "${mode_dir}" "${tuned_log}"
      tuned_status=${RUN_MODE_PROBE_STATUS}
      tuned_artifact_dir="$(artifact_dir_from_log "${tuned_log}" "${mode_dir}")"
      write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${tune_video_delay_us}" "${tune_audio_delay_us}" "${tuned_status}" "${tuned_log}" "${tuned_artifact_dir}" "${tuned_result}"
      cp "${tuned_result}" "${mode_result}"
      run_status=${tuned_status}
    else
      echo "   ↪ tune skipped: ${tune_reason:-not ready}"
    fi
  fi

  if [[ "${run_status}" -ne 0 && "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then
    break
  fi
done

summarize_matrix

echo "==> done"
echo "artifact_dir: ${MATRIX_REPORT_DIR}"
echo "mode_matrix_summary_json: ${MATRIX_SUMMARY_JSON}"
echo "mode_matrix_summary_csv: ${MATRIX_SUMMARY_CSV}"
echo "mode_matrix_summary_txt: ${MATRIX_SUMMARY_TXT}"
echo "mode_delay_recommendations_json: ${MATRIX_DELAY_JSON}"
echo "mode_delay_recommendations_env: ${MATRIX_DELAY_ENV}"

if python3 - <<'PY' "${MATRIX_SUMMARY_JSON}"
import json
import pathlib
import sys

summary = json.loads(pathlib.Path(sys.argv[1]).read_text())
raise SystemExit(0 if summary.get("passed") else 1)
PY
then
  exit 0
fi

exit 95