diff --git a/AGENTS.md b/AGENTS.md index b73ac71..ed02d37 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,6 +126,11 @@ path. - [ ] Keep UI/profile controls authoritative for UVC output profiles beyond `640x480@20`; validate `1280x720@30` and `1920x1080@20/30` after sync is locked. + - [x] Add a server-to-RC mode-matrix harness so the same sync/freshness/ + smoothness contract can be run against `640x480@20`, `1280x720@30`, + `1920x1080@20`, and `1920x1080@30`. + - [ ] Run the mode matrix on Theia/Tethys and record per-mode static delay + center points before changing the normal advertised profiles. - [ ] Keep the UI +/-5ms calibration nudges available as small post-baseline operator trims for future non-probeable remote hosts. - [x] Continue reporting client timing and sink handoff diagnostics from bundled packets. diff --git a/Cargo.lock b/Cargo.lock index 7ad72b4..c459ece 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.19.10" +version = "0.19.11" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.19.10" +version = "0.19.11" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.19.10" +version = "0.19.11" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 60e1a0a..c4ca9a5 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.19.10" +version = "0.19.11" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index d141b45..7c15bc5 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.19.10" +version = "0.19.11" edition = "2024" build = "build.rs" diff --git a/scripts/manual/run_server_to_rc_mode_matrix.sh b/scripts/manual/run_server_to_rc_mode_matrix.sh new file mode 100755 index 0000000..726e371 --- /dev/null +++ b/scripts/manual/run_server_to_rc_mode_matrix.sh @@ -0,0 +1,509 @@ +#!/usr/bin/env bash +# scripts/manual/run_server_to_rc_mode_matrix.sh +# Manual: validate server-generated UVC/UAC output against the RC target across +# the UVC modes the UI advertises. This is still a hardware-in-the-loop probe: +# it captures the real Tethys UVC/UAC endpoints and summarizes sync, +# freshness, and smoothness for each mode. + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)" + +TETHYS_HOST=${TETHYS_HOST:-tethys} +LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST:-theia} +LESAVKA_SERVER_CONNECT_HOST=${LESAVKA_SERVER_CONNECT_HOST:-38.28.125.112} +LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR:-auto} +LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https} +LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server} +LESAVKA_SERVER_REPO=${LESAVKA_SERVER_REPO:-/home/brad/Development/lesavka} +SSH_OPTS=${SSH_OPTS:-"-o BatchMode=yes -o ConnectTimeout=30"} + +LESAVKA_SERVER_RC_MODES=${LESAVKA_SERVER_RC_MODES:-640x480@20,1280x720@30,1920x1080@20,1920x1080@30} +LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US:-170000} +LESAVKA_SERVER_RC_MODE_DELAYS_US=${LESAVKA_SERVER_RC_MODE_DELAYS_US:-640x480@20=170000,1280x720@30=170000,1920x1080@20=170000,1920x1080@30=170000} +LESAVKA_SERVER_RC_RECONFIGURE=${LESAVKA_SERVER_RC_RECONFIGURE:-0} +LESAVKA_SERVER_RC_RECONFIGURE_REF=${LESAVKA_SERVER_RC_RECONFIGURE_REF:-master} +LESAVKA_SERVER_RC_RECONFIGURE_COMMAND=${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND:-} +LESAVKA_SERVER_RC_CONTINUE_ON_FAIL=${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL:-1} + +LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350} +LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS:-100} +LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250} +LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS:-1} +LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS=${LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS:-1} + +LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS=${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS:-0} +LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS=${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS:-0} +LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS=${LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS:-1} +LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS=${LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS:-1} +LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES:-12} +LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES:-12} +LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES:-5} +LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS=${LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS:-2} + +PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-20} +PROBE_WARMUP_SECONDS=${PROBE_WARMUP_SECONDS:-4} +PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,1,3,2,4,1,1,3,1,4,2,1,2,3,4,1,3,2,2,4,1,2,4,3,1,1,4,2,3,1,2} +REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst} +REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-cfr} +REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse} +REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto} +LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0} + +STAMP="$(date +%Y%m%d-%H%M%S)" +LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp} +MATRIX_REPORT_DIR=${MATRIX_REPORT_DIR:-"${LOCAL_OUTPUT_DIR%/}/lesavka-server-rc-mode-matrix-${STAMP}"} +MATRIX_SUMMARY_JSON="${MATRIX_REPORT_DIR}/mode-matrix-summary.json" +MATRIX_SUMMARY_CSV="${MATRIX_REPORT_DIR}/mode-matrix-summary.csv" +MATRIX_SUMMARY_TXT="${MATRIX_REPORT_DIR}/mode-matrix-summary.txt" +mkdir -p "${MATRIX_REPORT_DIR}" + +mode_id() { + local mode=$1 + printf '%s\n' "${mode//@/_}" | tr -c '[:alnum:]_.-' '_' +} + +parse_mode() { + local mode=$1 + if [[ ! "${mode}" =~ ^([0-9]+)x([0-9]+)@([0-9]+)$ ]]; then + printf 'invalid mode %s; expected WIDTHxHEIGHT@FPS\n' "${mode}" >&2 + exit 64 + fi + printf '%s %s %s\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}" +} + +lookup_video_delay_us() { + local mode=$1 + local entry key value + IFS=',' read -r -a delay_entries <<<"${LESAVKA_SERVER_RC_MODE_DELAYS_US}" + for entry in "${delay_entries[@]}"; do + key=${entry%%=*} + value=${entry#*=} + if [[ "${key}" == "${mode}" && "${value}" =~ ^-?[0-9]+$ ]]; then + printf '%s\n' "${value}" + return 0 + fi + done + printf '%s\n' "${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US}" +} + +reconfigure_server_mode() { + local mode=$1 + local width=$2 + local height=$3 + local fps=$4 + [[ "${LESAVKA_SERVER_RC_RECONFIGURE}" != "0" ]] || return 0 + + echo "==> reconfiguring ${LESAVKA_SERVER_HOST} UVC gadget for ${mode}" + if [[ -n "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}" ]]; then + LESAVKA_MODE="${mode}" \ + LESAVKA_UVC_WIDTH="${width}" \ + LESAVKA_UVC_HEIGHT="${height}" \ + LESAVKA_UVC_FPS="${fps}" \ + bash -c "${LESAVKA_SERVER_RC_RECONFIGURE_COMMAND}" + return 0 + fi + + ssh ${SSH_OPTS} "${LESAVKA_SERVER_HOST}" bash -s -- \ + "${LESAVKA_SERVER_REPO}" \ + "${LESAVKA_SERVER_RC_RECONFIGURE_REF}" \ + "${width}" \ + "${height}" \ + "${fps}" <<'REMOTE_RECONFIGURE' +set -euo pipefail +repo=$1 +ref=$2 +width=$3 +height=$4 +fps=$5 +cd "${repo}" +git fetch --all --prune +git checkout "${ref}" +git pull --ff-only +sudo env \ + LESAVKA_REF="${ref}" \ + LESAVKA_INSTALL_UVC_CODEC=mjpeg \ + LESAVKA_UVC_WIDTH="${width}" \ + LESAVKA_UVC_HEIGHT="${height}" \ + LESAVKA_UVC_FPS="${fps}" \ + LESAVKA_UVC_INTERVAL="$((10000000 / fps))" \ + ./scripts/install/server.sh +REMOTE_RECONFIGURE +} + +write_mode_result() { + local mode=$1 + local width=$2 + local height=$3 + local fps=$4 + local video_delay_us=$5 + local run_status=$6 + local run_log=$7 + local artifact_dir=$8 + local output_json=$9 + + python3 - <<'PY' \ + "${mode}" \ + "${width}" \ + "${height}" \ + "${fps}" \ + "${video_delay_us}" \ + "${run_status}" \ + "${run_log}" \ + "${artifact_dir}" \ + "${output_json}" \ + "${LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS}" \ + "${LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS}" \ + "${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS}" \ + "${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS}" \ + "${LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS}" \ + "${LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS}" \ + "${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES}" \ + "${LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES}" \ + "${LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES}" \ + "${LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS}" +import json +import math +import pathlib +import sys + +( + mode, + width_raw, + height_raw, + fps_raw, + video_delay_raw, + run_status_raw, + run_log, + artifact_dir_raw, + output_json, + require_sync_raw, + require_freshness_raw, + max_video_hiccups_raw, + max_audio_hiccups_raw, + max_video_jitter_raw, + max_audio_jitter_raw, + max_missing_raw, + max_undecodable_raw, + max_duplicates_raw, + max_low_rms_raw, +) = sys.argv[1:] + + +def as_bool(value): + return str(value).strip().lower() not in {"", "0", "false", "no", "off"} + + +def as_int(value, default=0): + try: + return int(str(value).strip()) + except Exception: + return default + + +def as_float(value, default=0.0): + try: + result = float(str(value).strip()) + except Exception: + return default + return result if math.isfinite(result) else default + + +def load_json(path): + try: + return json.loads(pathlib.Path(path).read_text()) + except Exception: + return {} + + +def nested(mapping, *keys, default=None): + value = mapping + for key in keys: + if not isinstance(value, dict): + return default + value = value.get(key) + return default if value is None else value + + +artifact_dir = pathlib.Path(artifact_dir_raw) if artifact_dir_raw else pathlib.Path() +report = load_json(artifact_dir / "report.json") +correlation = load_json(artifact_dir / "output-delay-correlation.json") +freshness = correlation.get("freshness") or {} +smoothness = correlation.get("smoothness") or {} +video = smoothness.get("video") or {} +audio = smoothness.get("audio") or {} +audio_cadence = audio.get("packet_cadence") or {} +audio_rms = audio.get("rms_continuity") or {} +verdict = report.get("verdict") or {} + +sync_pass = verdict.get("passed") is True +freshness_status = freshness.get("status", "unknown") +freshness_pass = freshness_status == "pass" +run_status = as_int(run_status_raw, 1) + +video_hiccups = as_int(video.get("hiccup_count"), 0) +audio_hiccups = as_int(audio_cadence.get("hiccup_count"), 0) +video_jitter = as_float(nested(video, "jitter_stats", "p95_jitter_ms"), 0.0) +audio_jitter = as_float(nested(audio_cadence, "jitter_stats", "p95_jitter_ms"), 0.0) +missing = as_int(video.get("estimated_missing_frames"), 0) +undecodable = as_int(video.get("undecodable_frames"), 0) +duplicates = as_int(video.get("duplicate_frames"), 0) +low_rms = as_int(audio_rms.get("low_rms_window_count"), 0) + +reasons = [] +if run_status != 0: + reasons.append(f"probe command exited {run_status}") +if as_bool(require_sync_raw) and not sync_pass: + reasons.append(f"sync did not pass: {verdict.get('status', 'unknown')}") +if as_bool(require_freshness_raw) and not freshness_pass: + reasons.append(f"freshness did not pass: {freshness_status}") +if video_hiccups > as_int(max_video_hiccups_raw): + reasons.append(f"video hiccups {video_hiccups} > {max_video_hiccups_raw}") +if audio_hiccups > as_int(max_audio_hiccups_raw): + reasons.append(f"audio hiccups {audio_hiccups} > {max_audio_hiccups_raw}") +if video_jitter > as_float(max_video_jitter_raw): + reasons.append(f"video p95 jitter {video_jitter:.1f}ms > {as_float(max_video_jitter_raw):.1f}ms") +if audio_jitter > as_float(max_audio_jitter_raw): + reasons.append(f"audio p95 jitter {audio_jitter:.1f}ms > {as_float(max_audio_jitter_raw):.1f}ms") +if missing > as_int(max_missing_raw): + reasons.append(f"estimated missing video frames {missing} > {max_missing_raw}") +if undecodable > as_int(max_undecodable_raw): + reasons.append(f"undecodable video frames {undecodable} > {max_undecodable_raw}") +if duplicates > as_int(max_duplicates_raw): + reasons.append(f"duplicate video frames {duplicates} > {max_duplicates_raw}") +if low_rms > as_int(max_low_rms_raw): + reasons.append(f"low-RMS audio windows {low_rms} > {max_low_rms_raw}") + +artifact = { + "schema": "lesavka.server-rc-mode-result.v1", + "mode": mode, + "width": as_int(width_raw), + "height": as_int(height_raw), + "fps": as_int(fps_raw), + "audio_delay_us": 0, + "video_delay_us": as_int(video_delay_raw), + "run_status": run_status, + "run_log": run_log, + "artifact_dir": str(artifact_dir), + "report_json": str(artifact_dir / "report.json"), + "correlation_json": str(artifact_dir / "output-delay-correlation.json"), + "passed": not reasons, + "failure_reasons": reasons, + "sync": { + "passed": sync_pass, + "status": verdict.get("status", "unknown"), + "reason": verdict.get("reason", ""), + "p95_abs_skew_ms": as_float(verdict.get("p95_abs_skew_ms"), 0.0), + "median_skew_ms": as_float(report.get("median_skew_ms"), 0.0), + "drift_ms": as_float(report.get("drift_ms"), 0.0), + "paired_event_count": as_int(report.get("paired_event_count"), 0), + }, + "freshness": { + "status": freshness_status, + "reason": freshness.get("reason", ""), + "worst_event_age_p95_ms": freshness.get("worst_event_age_p95_ms"), + "clock_uncertainty_ms": freshness.get("clock_uncertainty_ms"), + "worst_event_age_with_uncertainty_ms": freshness.get("worst_event_age_with_uncertainty_ms"), + "worst_freshness_drift_ms": freshness.get("worst_freshness_drift_ms"), + "max_age_limit_ms": freshness.get("max_age_limit_ms"), + }, + "smoothness": { + "video_frames": as_int(video.get("timestamps"), 0), + "video_p95_jitter_ms": video_jitter, + "video_max_interval_ms": as_float(nested(video, "interval_stats", "max_interval_ms"), 0.0), + "video_hiccups": video_hiccups, + "video_decoded_frames": as_int(video.get("decoded_frames"), 0), + "video_duplicate_frames": duplicates, + "video_estimated_missing_frames": missing, + "video_undecodable_frames": undecodable, + "audio_packets": as_int(audio_cadence.get("timestamps"), 0), + "audio_p95_jitter_ms": audio_jitter, + "audio_max_interval_ms": as_float(nested(audio_cadence, "interval_stats", "max_interval_ms"), 0.0), + "audio_hiccups": audio_hiccups, + "audio_low_rms_windows": low_rms, + "audio_median_rms": as_float(nested(audio_rms, "rms_stats", "median_rms"), 0.0), + }, +} +pathlib.Path(output_json).write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n") +PY +} + +summarize_matrix() { + python3 - <<'PY' "${MATRIX_REPORT_DIR}" "${MATRIX_SUMMARY_JSON}" "${MATRIX_SUMMARY_CSV}" "${MATRIX_SUMMARY_TXT}" +import csv +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +summary_json = pathlib.Path(sys.argv[2]) +summary_csv = pathlib.Path(sys.argv[3]) +summary_txt = pathlib.Path(sys.argv[4]) +results = [] +for path in sorted(root.glob("*/mode-result.json")): + try: + results.append(json.loads(path.read_text())) + except Exception: + continue + +summary = { + "schema": "lesavka.server-rc-mode-matrix-summary.v1", + "artifact_dir": str(root), + "passed": bool(results) and all(result.get("passed") for result in results), + "mode_count": len(results), + "results": results, +} +summary_json.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") + +fieldnames = [ + "mode", + "passed", + "video_delay_us", + "sync_status", + "p95_abs_skew_ms", + "median_skew_ms", + "sync_drift_ms", + "freshness_status", + "freshness_budget_ms", + "freshness_drift_ms", + "video_hiccups", + "video_missing", + "video_undecodable", + "audio_hiccups", + "audio_low_rms", + "artifact_dir", +] +with summary_csv.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + for result in results: + writer.writerow({ + "mode": result.get("mode"), + "passed": result.get("passed"), + "video_delay_us": result.get("video_delay_us"), + "sync_status": (result.get("sync") or {}).get("status"), + "p95_abs_skew_ms": (result.get("sync") or {}).get("p95_abs_skew_ms"), + "median_skew_ms": (result.get("sync") or {}).get("median_skew_ms"), + "sync_drift_ms": (result.get("sync") or {}).get("drift_ms"), + "freshness_status": (result.get("freshness") or {}).get("status"), + "freshness_budget_ms": (result.get("freshness") or {}).get("worst_event_age_with_uncertainty_ms"), + "freshness_drift_ms": (result.get("freshness") or {}).get("worst_freshness_drift_ms"), + "video_hiccups": (result.get("smoothness") or {}).get("video_hiccups"), + "video_missing": (result.get("smoothness") or {}).get("video_estimated_missing_frames"), + "video_undecodable": (result.get("smoothness") or {}).get("video_undecodable_frames"), + "audio_hiccups": (result.get("smoothness") or {}).get("audio_hiccups"), + "audio_low_rms": (result.get("smoothness") or {}).get("audio_low_rms_windows"), + "artifact_dir": result.get("artifact_dir"), + }) + +lines = [ + f"Server-to-RC mode matrix for {root}", + f"- modes: {len(results)}", + f"- verdict: {'pass' if summary['passed'] else 'fail'}", +] +for result in results: + sync = result.get("sync") or {} + freshness = result.get("freshness") or {} + smooth = result.get("smoothness") or {} + marker = "PASS" if result.get("passed") else "FAIL" + lines.append( + f"- {marker} {result.get('mode')}: " + f"sync {sync.get('status')} p95={sync.get('p95_abs_skew_ms', 0.0):.1f}ms median={sync.get('median_skew_ms', 0.0):+.1f}ms; " + f"freshness {freshness.get('status')} budget={freshness.get('worst_event_age_with_uncertainty_ms') or 0.0:.1f}ms; " + f"smooth video hiccups={smooth.get('video_hiccups', 0)} missing={smooth.get('video_estimated_missing_frames', 0)} undecodable={smooth.get('video_undecodable_frames', 0)} audio hiccups={smooth.get('audio_hiccups', 0)}" + ) + for reason in result.get("failure_reasons") or []: + lines.append(f" reason: {reason}") +summary_txt.write_text("\n".join(lines) + "\n") +print("\n".join(lines)) +PY +} + +echo "==> server-to-RC mode matrix" +echo " ↪ modes=${LESAVKA_SERVER_RC_MODES}" +echo " ↪ delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}" +echo " ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" +echo " ↪ artifact_dir=${MATRIX_REPORT_DIR}" + +IFS=',' read -r -a modes <<<"${LESAVKA_SERVER_RC_MODES}" +for mode in "${modes[@]}"; do + mode="${mode//[[:space:]]/}" + [[ -n "${mode}" ]] || continue + read -r width height fps < <(parse_mode "${mode}") + video_delay_us="$(lookup_video_delay_us "${mode}")" + id="$(mode_id "${mode}")" + mode_dir="${MATRIX_REPORT_DIR}/${id}" + mode_log="${mode_dir}/mode-run.log" + mode_result="${mode_dir}/mode-result.json" + mkdir -p "${mode_dir}" + + echo "==> mode ${mode}: video_delay_us=${video_delay_us}" + reconfigure_server_mode "${mode}" "${width}" "${height}" "${fps}" + + set +e + TETHYS_HOST="${TETHYS_HOST}" \ + LESAVKA_SERVER_HOST="${LESAVKA_SERVER_HOST}" \ + LESAVKA_SERVER_CONNECT_HOST="${LESAVKA_SERVER_CONNECT_HOST}" \ + LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_ADDR}" \ + LESAVKA_SERVER_SCHEME="${LESAVKA_SERVER_SCHEME}" \ + LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \ + SSH_OPTS="${SSH_OPTS}" \ + REMOTE_PULSE_CAPTURE_TOOL="${REMOTE_PULSE_CAPTURE_TOOL}" \ + REMOTE_PULSE_VIDEO_MODE="${REMOTE_PULSE_VIDEO_MODE}" \ + REMOTE_CAPTURE_STACK="${REMOTE_CAPTURE_STACK}" \ + REMOTE_AUDIO_SOURCE="${REMOTE_AUDIO_SOURCE}" \ + VIDEO_SIZE="${width}x${height}" \ + VIDEO_FPS="${fps}" \ + VIDEO_FORMAT=mjpeg \ + REMOTE_EXPECT_UVC_WIDTH="${width}" \ + REMOTE_EXPECT_UVC_HEIGHT="${height}" \ + REMOTE_EXPECT_UVC_FPS="${fps}" \ + LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US="${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US}" \ + LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US="${video_delay_us}" \ + LESAVKA_OUTPUT_DELAY_APPLY=0 \ + LESAVKA_OUTPUT_DELAY_SAVE=0 \ + LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0 \ + LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" \ + LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}" \ + LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}" \ + PROBE_EVENT_WIDTH_CODES="${PROBE_EVENT_WIDTH_CODES}" \ + PROBE_DURATION_SECONDS="${PROBE_DURATION_SECONDS}" \ + PROBE_WARMUP_SECONDS="${PROBE_WARMUP_SECONDS}" \ + LOCAL_OUTPUT_DIR="${mode_dir}" \ + "${SCRIPT_DIR}/run_upstream_av_sync.sh" 2>&1 | tee "${mode_log}" + run_status=${PIPESTATUS[0]} + set -e + + artifact_dir="$(awk -F': ' '/^artifact_dir: / {print $2}' "${mode_log}" | tail -n1)" + if [[ -z "${artifact_dir}" ]]; then + artifact_dir="$(find "${mode_dir}" -mindepth 1 -maxdepth 1 -type d -name 'lesavka-output-delay-probe-*' | sort | tail -n1 || true)" + fi + write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${video_delay_us}" "${run_status}" "${mode_log}" "${artifact_dir}" "${mode_result}" + + if [[ "${run_status}" -ne 0 && "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then + break + fi +done + +summarize_matrix + +echo "==> done" +echo "artifact_dir: ${MATRIX_REPORT_DIR}" +echo "mode_matrix_summary_json: ${MATRIX_SUMMARY_JSON}" +echo "mode_matrix_summary_csv: ${MATRIX_SUMMARY_CSV}" +echo "mode_matrix_summary_txt: ${MATRIX_SUMMARY_TXT}" + +if python3 - <<'PY' "${MATRIX_SUMMARY_JSON}" +import json +import pathlib +import sys + +summary = json.loads(pathlib.Path(sys.argv[1]).read_text()) +raise SystemExit(0 if summary.get("passed") else 1) +PY +then + exit 0 +fi + +exit 95 diff --git a/scripts/manual/run_upstream_av_sync.sh b/scripts/manual/run_upstream_av_sync.sh index 87227c0..6bd9046 100755 --- a/scripts/manual/run_upstream_av_sync.sh +++ b/scripts/manual/run_upstream_av_sync.sh @@ -53,6 +53,9 @@ FETCH_CAPTURE=${FETCH_CAPTURE:-1} REMOTE_SERVER_PREFLIGHT=${REMOTE_SERVER_PREFLIGHT:-1} REMOTE_EXPECT_CAM_OUTPUT=${REMOTE_EXPECT_CAM_OUTPUT:-uvc} REMOTE_EXPECT_UVC_CODEC=${REMOTE_EXPECT_UVC_CODEC:-mjpeg} +REMOTE_EXPECT_UVC_WIDTH=${REMOTE_EXPECT_UVC_WIDTH:-} +REMOTE_EXPECT_UVC_HEIGHT=${REMOTE_EXPECT_UVC_HEIGHT:-} +REMOTE_EXPECT_UVC_FPS=${REMOTE_EXPECT_UVC_FPS:-} LESAVKA_OUTPUT_DELAY_CALIBRATION=${LESAVKA_OUTPUT_DELAY_CALIBRATION:-1} LESAVKA_OUTPUT_DELAY_APPLY=${LESAVKA_OUTPUT_DELAY_APPLY:-0} LESAVKA_OUTPUT_DELAY_APPLY_MODE=${LESAVKA_OUTPUT_DELAY_APPLY_MODE:-absolute} @@ -522,10 +525,16 @@ preflight_server_path() { echo "==> verifying Lesavka server path on ${LESAVKA_SERVER_HOST}" ssh ${SSH_OPTS} "${LESAVKA_SERVER_HOST}" bash -s -- \ "${REMOTE_EXPECT_CAM_OUTPUT}" \ - "${REMOTE_EXPECT_UVC_CODEC}" <<'REMOTE_PREFLIGHT' + "${REMOTE_EXPECT_UVC_CODEC}" \ + "${REMOTE_EXPECT_UVC_WIDTH}" \ + "${REMOTE_EXPECT_UVC_HEIGHT}" \ + "${REMOTE_EXPECT_UVC_FPS}" <<'REMOTE_PREFLIGHT' set -euo pipefail expect_cam_output=$1 expect_uvc_codec=$2 +expect_uvc_width=$3 +expect_uvc_height=$4 +expect_uvc_fps=$5 read_env_value() { local key=$1 @@ -538,10 +547,18 @@ read_env_value() { cam_output=$(read_env_value "LESAVKA_CAM_OUTPUT" /etc/lesavka/server.env) server_uvc_codec=$(read_env_value "LESAVKA_UVC_CODEC" /etc/lesavka/server.env) runtime_uvc_codec=$(read_env_value "LESAVKA_UVC_CODEC" /etc/lesavka/uvc.env) +server_uvc_width=$(read_env_value "LESAVKA_UVC_WIDTH" /etc/lesavka/server.env) +server_uvc_height=$(read_env_value "LESAVKA_UVC_HEIGHT" /etc/lesavka/server.env) +server_uvc_fps=$(read_env_value "LESAVKA_UVC_FPS" /etc/lesavka/server.env) +runtime_uvc_width=$(read_env_value "LESAVKA_UVC_WIDTH" /etc/lesavka/uvc.env) +runtime_uvc_height=$(read_env_value "LESAVKA_UVC_HEIGHT" /etc/lesavka/uvc.env) +runtime_uvc_fps=$(read_env_value "LESAVKA_UVC_FPS" /etc/lesavka/uvc.env) printf ' ↪ server.env CAM_OUTPUT=%s\n' "${cam_output:-}" printf ' ↪ server.env UVC_CODEC=%s\n' "${server_uvc_codec:-}" printf ' ↪ uvc.env UVC_CODEC=%s\n' "${runtime_uvc_codec:-}" +printf ' ↪ server.env UVC_MODE=%sx%s@%s\n' "${server_uvc_width:-}" "${server_uvc_height:-}" "${server_uvc_fps:-}" +printf ' ↪ uvc.env UVC_MODE=%sx%s@%s\n' "${runtime_uvc_width:-}" "${runtime_uvc_height:-}" "${runtime_uvc_fps:-}" if [[ -n "${expect_cam_output}" && "${cam_output}" != "${expect_cam_output}" ]]; then printf 'expected CAM_OUTPUT=%s but found %s\n' "${expect_cam_output}" "${cam_output:-}" >&2 @@ -555,6 +572,30 @@ if [[ -n "${expect_uvc_codec}" && "${runtime_uvc_codec}" != "${expect_uvc_codec} printf 'expected uvc.env UVC_CODEC=%s but found %s\n' "${expect_uvc_codec}" "${runtime_uvc_codec:-}" >&2 exit 66 fi +if [[ -n "${expect_uvc_width}" && "${server_uvc_width}" != "${expect_uvc_width}" ]]; then + printf 'expected server.env UVC_WIDTH=%s but found %s\n' "${expect_uvc_width}" "${server_uvc_width:-}" >&2 + exit 67 +fi +if [[ -n "${expect_uvc_height}" && "${server_uvc_height}" != "${expect_uvc_height}" ]]; then + printf 'expected server.env UVC_HEIGHT=%s but found %s\n' "${expect_uvc_height}" "${server_uvc_height:-}" >&2 + exit 68 +fi +if [[ -n "${expect_uvc_fps}" && "${server_uvc_fps}" != "${expect_uvc_fps}" ]]; then + printf 'expected server.env UVC_FPS=%s but found %s\n' "${expect_uvc_fps}" "${server_uvc_fps:-}" >&2 + exit 69 +fi +if [[ -n "${expect_uvc_width}" && "${runtime_uvc_width}" != "${expect_uvc_width}" ]]; then + printf 'expected uvc.env UVC_WIDTH=%s but found %s\n' "${expect_uvc_width}" "${runtime_uvc_width:-}" >&2 + exit 70 +fi +if [[ -n "${expect_uvc_height}" && "${runtime_uvc_height}" != "${expect_uvc_height}" ]]; then + printf 'expected uvc.env UVC_HEIGHT=%s but found %s\n' "${expect_uvc_height}" "${runtime_uvc_height:-}" >&2 + exit 71 +fi +if [[ -n "${expect_uvc_fps}" && "${runtime_uvc_fps}" != "${expect_uvc_fps}" ]]; then + printf 'expected uvc.env UVC_FPS=%s but found %s\n' "${expect_uvc_fps}" "${runtime_uvc_fps:-}" >&2 + exit 72 +fi systemctl is-active lesavka-server lesavka-uvc lesavka-core >/dev/null REMOTE_PREFLIGHT diff --git a/server/Cargo.toml b/server/Cargo.toml index df461cf..6cc4c70 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.19.10" +version = "0.19.11" edition = "2024" autobins = false diff --git a/testing/tests/client_manual_sync_script_contract.rs b/testing/tests/client_manual_sync_script_contract.rs index 3ac61e3..4cf9b6b 100644 --- a/testing/tests/client_manual_sync_script_contract.rs +++ b/testing/tests/client_manual_sync_script_contract.rs @@ -6,6 +6,8 @@ //! port is not exposed on the public SSH endpoint. const SYNC_SCRIPT: &str = include_str!("../../scripts/manual/run_upstream_av_sync.sh"); +const SERVER_RC_MODE_MATRIX_SCRIPT: &str = + include_str!("../../scripts/manual/run_server_to_rc_mode_matrix.sh"); const BROWSER_SYNC_SCRIPT: &str = include_str!("../../scripts/manual/run_upstream_browser_av_sync.sh"); const MIRRORED_SYNC_SCRIPT: &str = @@ -54,6 +56,13 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() { "LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS:-1000}", "LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS:-100}", "LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}", + "REMOTE_EXPECT_UVC_WIDTH=${REMOTE_EXPECT_UVC_WIDTH:-}", + "REMOTE_EXPECT_UVC_HEIGHT=${REMOTE_EXPECT_UVC_HEIGHT:-}", + "REMOTE_EXPECT_UVC_FPS=${REMOTE_EXPECT_UVC_FPS:-}", + "server.env UVC_MODE=", + "uvc.env UVC_MODE=", + "expected server.env UVC_WIDTH", + "expected uvc.env UVC_FPS", "server-to-capture clock alignment unavailable; falling back to client-mediated SSH samples", "LESAVKA_CLOCK_ALIGNMENT_SAMPLES=${LESAVKA_CLOCK_ALIGNMENT_SAMPLES:-5}", "sample_best_host_clock_offset_ns", @@ -174,6 +183,47 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() { } } +#[test] +fn server_rc_mode_matrix_validates_advertised_uvc_profiles() { + for expected in [ + "LESAVKA_SERVER_RC_MODES=${LESAVKA_SERVER_RC_MODES:-640x480@20,1280x720@30,1920x1080@20,1920x1080@30}", + "LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US:-170000}", + "LESAVKA_SERVER_RC_MODE_DELAYS_US=${LESAVKA_SERVER_RC_MODE_DELAYS_US:-640x480@20=170000,1280x720@30=170000,1920x1080@20=170000,1920x1080@30=170000}", + "LESAVKA_SERVER_RC_RECONFIGURE=${LESAVKA_SERVER_RC_RECONFIGURE:-0}", + "LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350}", + "LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS=${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS:-0}", + "LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS=${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS:-0}", + "LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES:-12}", + "mode-matrix-summary.json", + "mode-matrix-summary.csv", + "mode-matrix-summary.txt", + "schema\": \"lesavka.server-rc-mode-result.v1\"", + "schema\": \"lesavka.server-rc-mode-matrix-summary.v1\"", + "REMOTE_PULSE_CAPTURE_TOOL=\"${REMOTE_PULSE_CAPTURE_TOOL}\"", + "REMOTE_PULSE_VIDEO_MODE=\"${REMOTE_PULSE_VIDEO_MODE}\"", + "VIDEO_SIZE=\"${width}x${height}\"", + "VIDEO_FPS=\"${fps}\"", + "REMOTE_EXPECT_UVC_WIDTH=\"${width}\"", + "REMOTE_EXPECT_UVC_HEIGHT=\"${height}\"", + "REMOTE_EXPECT_UVC_FPS=\"${fps}\"", + "LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=\"${video_delay_us}\"", + "LESAVKA_OUTPUT_DELAY_APPLY=0", + "LESAVKA_OUTPUT_DELAY_SAVE=0", + "LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=\"${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}\"", + "LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=\"${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}\"", + "sync did not pass", + "freshness did not pass", + "video hiccups", + "estimated missing video frames", + "audio hiccups", + ] { + assert!( + SERVER_RC_MODE_MATRIX_SCRIPT.contains(expected), + "server-to-RC mode matrix script should contain {expected}" + ); + } +} + #[test] fn browser_sync_script_can_delegate_to_a_real_path_driver() { for expected in [