2026-05-04 00:47:21 -03:00
#!/usr/bin/env bash
# scripts/manual/run_server_to_rc_mode_matrix.sh
# Manual: validate server-generated UVC/UAC output against the RC target across
# the UVC modes the UI advertises. This is still a hardware-in-the-loop probe:
# it captures the real Tethys UVC/UAC endpoints and summarizes sync,
# freshness, and smoothness for each mode.
2026-05-06 05:50:59 -03:00
# Not part of CI: it needs the Theia/Tethys lab hosts and live USB gadget state.
2026-05-04 12:48:17 -03:00
#
# Reconfigure mode is intentionally a fast runtime path: it updates the remote
# Lesavka env files and cycles the UVC gadget, but it does not rebuild or
# reinstall the server binary for each mode.
2026-05-04 00:47:21 -03:00
set -euo pipefail
SCRIPT_DIR = " $( cd -- " $( dirname " ${ BASH_SOURCE [0] } " ) " >/dev/null 2>& 1 && pwd ) "
REPO_ROOT = " $( cd -- " ${ SCRIPT_DIR } /../.. " >/dev/null 2>& 1 && pwd ) "
2026-05-04 01:22:25 -03:00
if ( ( EUID = = 0 ) ) ; then
printf 'Do not run this matrix with local sudo.\n' >& 2
printf 'Run it as your normal workstation user; the script will request sudo on the remote server host only when reconfiguring UVC.\n' >& 2
exit 64
fi
2026-05-04 00:47:21 -03:00
TETHYS_HOST = ${ TETHYS_HOST :- tethys }
LESAVKA_SERVER_HOST = ${ LESAVKA_SERVER_HOST :- theia }
LESAVKA_SERVER_CONNECT_HOST = ${ LESAVKA_SERVER_CONNECT_HOST :- 38 .28.125.112 }
LESAVKA_SERVER_ADDR = ${ LESAVKA_SERVER_ADDR :- auto }
LESAVKA_SERVER_SCHEME = ${ LESAVKA_SERVER_SCHEME :- https }
LESAVKA_TLS_DOMAIN = ${ LESAVKA_TLS_DOMAIN :- lesavka -server }
2026-05-04 01:01:08 -03:00
LESAVKA_SERVER_REPO = ${ LESAVKA_SERVER_REPO :- auto }
2026-05-04 00:47:21 -03:00
SSH_OPTS = ${ SSH_OPTS :- "-o BatchMode=yes -o ConnectTimeout=30" }
2026-05-04 12:48:17 -03:00
LESAVKA_SERVER_RC_CORE_WEBCAM_MODES = ${ LESAVKA_SERVER_RC_CORE_WEBCAM_MODES :- 1280x720 @20,1280x720@30,1920x1080@20,1920x1080@30 }
LESAVKA_SERVER_RC_MODES = ${ LESAVKA_SERVER_RC_MODES :- ${ LESAVKA_SERVER_RC_CORE_WEBCAM_MODES } }
LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US = ${ LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US :- ${ LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US :- 0 } }
2026-05-06 03:59:20 -03:00
LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US = ${ LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US :- 135090 }
2026-05-04 12:48:17 -03:00
LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US = ${ LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US :- 1280x720 @20= ${ LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US } ,1280x720@30= ${ LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US } ,1920x1080@20= ${ LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US } ,1920x1080@30= ${ LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US } }
2026-05-06 03:59:20 -03:00
LESAVKA_SERVER_RC_MODE_DELAYS_US = ${ LESAVKA_SERVER_RC_MODE_DELAYS_US :- 1280x720 @20=162659,1280x720@30=135090,1920x1080@20=160045,1920x1080@30=127952 }
2026-05-04 12:48:17 -03:00
LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES = ${ LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES :- 1280x720 ,1920x1080 }
LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS = ${ LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS :- 20 ,30 }
LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX = ${ LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX :- Logitech |BRIO|C9[0-9]+|HD UVC WebCam|USB2[.]0 HD|Integrated Camera|Webcam|Camera }
LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX = ${ LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX :- Lesavka |UGREEN|MACROSILICON|Composite|Capture }
LESAVKA_SERVER_RC_MODE_SOURCE = ${ LESAVKA_SERVER_RC_MODE_SOURCE :- configured }
2026-05-04 00:47:21 -03:00
LESAVKA_SERVER_RC_RECONFIGURE = ${ LESAVKA_SERVER_RC_RECONFIGURE :- 0 }
LESAVKA_SERVER_RC_RECONFIGURE_REF = ${ LESAVKA_SERVER_RC_RECONFIGURE_REF :- master }
2026-05-04 01:12:20 -03:00
LESAVKA_SERVER_RC_RECONFIGURE_UPDATE = ${ LESAVKA_SERVER_RC_RECONFIGURE_UPDATE :- 0 }
2026-05-04 00:47:21 -03:00
LESAVKA_SERVER_RC_RECONFIGURE_COMMAND = ${ LESAVKA_SERVER_RC_RECONFIGURE_COMMAND :- }
2026-05-04 12:48:17 -03:00
LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY = ${ LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY :- runtime }
LESAVKA_SERVER_RC_RECONFIGURE_CODEC = ${ LESAVKA_SERVER_RC_RECONFIGURE_CODEC :- mjpeg }
LESAVKA_SERVER_RC_ALLOW_GADGET_RESET = ${ LESAVKA_SERVER_RC_ALLOW_GADGET_RESET :- 1 }
LESAVKA_SERVER_RC_FORCE_GADGET_REBUILD = ${ LESAVKA_SERVER_RC_FORCE_GADGET_REBUILD :- 1 }
LESAVKA_SERVER_RC_RECONFIGURE_SETTLE_SECONDS = ${ LESAVKA_SERVER_RC_RECONFIGURE_SETTLE_SECONDS :- 4 }
LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE = ${ LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE :- 0 }
LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY = ${ LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY :- 1 }
LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD = ${ LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD :- }
2026-05-04 20:32:21 -03:00
LESAVKA_SERVER_RC_START_DELAY_SECONDS = ${ LESAVKA_SERVER_RC_START_DELAY_SECONDS :- 0 }
2026-05-04 12:48:17 -03:00
LESAVKA_SERVER_RC_WAIT_TETHYS_READY = ${ LESAVKA_SERVER_RC_WAIT_TETHYS_READY :- 1 }
LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS = ${ LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS :- 60 }
LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS = ${ LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS :- 6 }
LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS = ${ LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS :- 3 }
LESAVKA_SERVER_RC_PROBE_PREBUILD = ${ LESAVKA_SERVER_RC_PROBE_PREBUILD :- 1 }
2026-05-04 00:47:21 -03:00
LESAVKA_SERVER_RC_CONTINUE_ON_FAIL = ${ LESAVKA_SERVER_RC_CONTINUE_ON_FAIL :- 1 }
2026-05-04 14:35:33 -03:00
LESAVKA_SERVER_RC_TUNE_DELAYS = ${ LESAVKA_SERVER_RC_TUNE_DELAYS :- 1 }
LESAVKA_SERVER_RC_TUNE_CONFIRM = ${ LESAVKA_SERVER_RC_TUNE_CONFIRM :- 1 }
2026-05-04 15:52:51 -03:00
LESAVKA_SERVER_RC_TUNE_MIN_PAIRS = ${ LESAVKA_SERVER_RC_TUNE_MIN_PAIRS :- 13 }
2026-05-04 14:50:47 -03:00
LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS = ${ LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS :- 1000 }
2026-05-04 14:35:33 -03:00
LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS = ${ LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS :- 80 }
2026-05-04 14:50:47 -03:00
LESAVKA_SERVER_RC_TUNE_MAX_STEP_US = ${ LESAVKA_SERVER_RC_TUNE_MAX_STEP_US :- 500000 }
2026-05-04 14:35:33 -03:00
LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US = ${ LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US :- 5000 }
2026-05-06 00:29:00 -03:00
LESAVKA_SERVER_RC_REPEAT_COUNT = ${ LESAVKA_SERVER_RC_REPEAT_COUNT :- 1 }
LESAVKA_SERVER_RC_VERBOSE_PROBES = ${ LESAVKA_SERVER_RC_VERBOSE_PROBES :- 1 }
LESAVKA_SERVER_RC_STATIC_MIN_RUNS = ${ LESAVKA_SERVER_RC_STATIC_MIN_RUNS :- 3 }
LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US = ${ LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US :- 30000 }
LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS = ${ LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS :- 35 }
LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS = ${ LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS :- 20 }
LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS = ${ LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS :- 1 }
LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS = ${ LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS :- 0 }
2026-05-04 00:47:21 -03:00
LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS = ${ LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS :- 350 }
LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS = ${ LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS :- 100 }
LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS = ${ LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS :- 250 }
2026-05-04 20:22:57 -03:00
LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS = ${ LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS :- ${ LESAVKA_SERVER_RC_TUNE_MIN_PAIRS } }
2026-05-04 00:47:21 -03:00
LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS = ${ LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS :- 1 }
LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS = ${ LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS :- 1 }
2026-05-05 19:33:56 -03:00
LESAVKA_SERVER_RC_MIN_CODED_PAIRS = ${ LESAVKA_SERVER_RC_MIN_CODED_PAIRS :- ${ LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS } }
LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS = ${ LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS :- 0 }
LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS = ${ LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS :- 0 }
LESAVKA_SERVER_RC_SIGNAL_READY = ${ LESAVKA_SERVER_RC_SIGNAL_READY :- 1 }
2026-05-05 21:01:05 -03:00
LESAVKA_SERVER_RC_SIGNAL_READY_MODE = ${ LESAVKA_SERVER_RC_SIGNAL_READY_MODE :- conditioned_capture }
2026-05-05 19:33:56 -03:00
LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS = ${ LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS :- 3 }
2026-05-05 20:27:37 -03:00
LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS = ${ LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS :- 12 }
2026-05-05 19:33:56 -03:00
LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS = ${ LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS :- 1 }
2026-05-05 20:27:37 -03:00
LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS = ${ LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS :- 4 }
LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS = ${ LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS :- 5 }
2026-05-05 21:01:05 -03:00
LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS = ${ LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS :- 12 }
LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS = ${ LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS :- 1 }
LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS = ${ LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS :- 1 }
LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES = ${ LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES :- ${ PROBE_EVENT_WIDTH_CODES :- 1 ,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 } }
LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW = ${ LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW :- auto }
2026-05-04 00:47:21 -03:00
LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS = ${ LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS :- 0 }
LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS = ${ LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS :- 0 }
LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS = ${ LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS :- 1 }
LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS = ${ LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS :- 1 }
LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES = ${ LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES :- 12 }
LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES = ${ LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES :- 12 }
LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES = ${ LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES :- 5 }
LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS = ${ LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS :- 2 }
PROBE_DURATION_SECONDS = ${ PROBE_DURATION_SECONDS :- 20 }
PROBE_WARMUP_SECONDS = ${ PROBE_WARMUP_SECONDS :- 4 }
2026-05-04 15:50:26 -03:00
PROBE_EVENT_WIDTH_CODES = ${ PROBE_EVENT_WIDTH_CODES :- 1 ,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 }
2026-05-04 00:47:21 -03:00
REMOTE_PULSE_CAPTURE_TOOL = ${ REMOTE_PULSE_CAPTURE_TOOL :- gst }
REMOTE_PULSE_VIDEO_MODE = ${ REMOTE_PULSE_VIDEO_MODE :- cfr }
REMOTE_CAPTURE_STACK = ${ REMOTE_CAPTURE_STACK :- pulse }
REMOTE_AUDIO_SOURCE = ${ REMOTE_AUDIO_SOURCE :- auto }
2026-05-04 14:05:55 -03:00
REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK = ${ REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK :- 0 }
2026-05-04 16:58:55 -03:00
REMOTE_CAPTURE_READY_SETTLE_SECONDS = ${ REMOTE_CAPTURE_READY_SETTLE_SECONDS :- 1 }
2026-05-04 00:47:21 -03:00
LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US = ${ LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US :- 0 }
STAMP = " $( date +%Y%m%d-%H%M%S) "
LOCAL_OUTPUT_DIR = ${ LOCAL_OUTPUT_DIR :- /tmp }
MATRIX_REPORT_DIR = ${ MATRIX_REPORT_DIR :- " ${ LOCAL_OUTPUT_DIR %/ } /lesavka-server-rc-mode-matrix- ${ STAMP } " }
MATRIX_SUMMARY_JSON = " ${ MATRIX_REPORT_DIR } /mode-matrix-summary.json "
MATRIX_SUMMARY_CSV = " ${ MATRIX_REPORT_DIR } /mode-matrix-summary.csv "
MATRIX_SUMMARY_TXT = " ${ MATRIX_REPORT_DIR } /mode-matrix-summary.txt "
2026-05-04 14:35:33 -03:00
MATRIX_DELAY_JSON = " ${ MATRIX_REPORT_DIR } /mode-delay-recommendations.json "
MATRIX_DELAY_ENV = " ${ MATRIX_REPORT_DIR } /mode-delay-recommendations.env "
2026-05-06 00:29:00 -03:00
MATRIX_STATIC_JSON = " ${ MATRIX_REPORT_DIR } /mode-static-calibration.json "
MATRIX_STATIC_CSV = " ${ MATRIX_REPORT_DIR } /mode-static-calibration.csv "
MATRIX_STATIC_TXT = " ${ MATRIX_REPORT_DIR } /mode-static-calibration.txt "
MATRIX_STATIC_ENV = " ${ MATRIX_REPORT_DIR } /mode-static-calibration.env "
MATRIX_RUN_LOG = " ${ MATRIX_REPORT_DIR } /mode-matrix-run.log "
2026-05-04 00:47:21 -03:00
mkdir -p " ${ MATRIX_REPORT_DIR } "
2026-05-06 00:29:00 -03:00
exec > >( tee -a " ${ MATRIX_RUN_LOG } " ) 2>& 1
2026-05-04 00:47:21 -03:00
mode_id( ) {
local mode = $1
printf '%s\n' " ${ mode //@/_ } " | tr -c '[:alnum:]_.-' '_'
}
parse_mode( ) {
local mode = $1
if [ [ ! " ${ mode } " = ~ ^( [ 0-9] +) x( [ 0-9] +) @( [ 0-9] +) $ ] ] ; then
printf 'invalid mode %s; expected WIDTHxHEIGHT@FPS\n' " ${ mode } " >& 2
exit 64
fi
printf '%s %s %s\n' " ${ BASH_REMATCH [1] } " " ${ BASH_REMATCH [2] } " " ${ BASH_REMATCH [3] } "
}
2026-05-04 12:48:17 -03:00
lookup_mode_delay_us( ) {
2026-05-04 00:47:21 -03:00
local mode = $1
2026-05-04 12:48:17 -03:00
local delay_map = $2
local default_value = $3
2026-05-04 00:47:21 -03:00
local entry key value
2026-05-04 12:48:17 -03:00
IFS = ',' read -r -a delay_entries <<< " ${ delay_map } "
2026-05-04 00:47:21 -03:00
for entry in " ${ delay_entries [@] } " ; do
key = ${ entry %%=* }
value = ${ entry #*= }
if [ [ " ${ key } " = = " ${ mode } " && " ${ value } " = ~ ^-?[ 0-9] +$ ] ] ; then
printf '%s\n' " ${ value } "
return 0
fi
done
2026-05-04 12:48:17 -03:00
printf '%s\n' " ${ default_value } "
}
lookup_audio_delay_us( ) {
lookup_mode_delay_us " $1 " " ${ LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US } " " ${ LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US } "
}
lookup_video_delay_us( ) {
lookup_mode_delay_us " $1 " " ${ LESAVKA_SERVER_RC_MODE_DELAYS_US } " " ${ LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US } "
}
2026-05-04 14:35:33 -03:00
artifact_dir_from_log( ) {
local log_path = $1
local search_dir = $2
local artifact_dir
artifact_dir = " $( awk -F': ' '/^artifact_dir: / {print $2}' " ${ log_path } " 2>/dev/null | tail -n1 || true ) "
if [ [ -z " ${ artifact_dir } " ] ] ; then
artifact_dir = " $( find " ${ search_dir } " -mindepth 1 -maxdepth 1 -type d -name 'lesavka-output-delay-probe-*' | sort | tail -n1 || true ) "
fi
printf '%s\n' " ${ artifact_dir } "
}
RUN_MODE_PROBE_STATUS = 0
run_mode_probe( ) {
local width = $1
local height = $2
local fps = $3
local audio_delay_us = $4
local video_delay_us = $5
local output_dir = $6
local log_path = $7
2026-05-05 19:33:56 -03:00
local probe_duration_seconds = ${ 8 :- ${ PROBE_DURATION_SECONDS } }
local probe_warmup_seconds = ${ 9 :- ${ PROBE_WARMUP_SECONDS } }
local min_pairs = ${ 10 :- ${ LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS } }
2026-05-05 21:01:05 -03:00
local conditioning_seconds = ${ 11 :- ${ LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS } }
local analysis_timeline_window = ${ LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW }
if [ [ " ${ LESAVKA_SERVER_RC_SIGNAL_READY } " = = "0" ] ] ; then
conditioning_seconds = 0
fi
if [ [ " ${ analysis_timeline_window } " = = "auto" ] ] ; then
if [ [ " ${ conditioning_seconds } " = ~ ^[ 0-9] +$ && " ${ conditioning_seconds } " -gt 0 ] ] ; then
analysis_timeline_window = 1
else
analysis_timeline_window = 0
fi
fi
2026-05-04 14:35:33 -03:00
2026-05-06 00:29:00 -03:00
local -a probe_env = (
" TETHYS_HOST= ${ TETHYS_HOST } "
" LESAVKA_SERVER_HOST= ${ LESAVKA_SERVER_HOST } "
" LESAVKA_SERVER_CONNECT_HOST= ${ LESAVKA_SERVER_CONNECT_HOST } "
" LESAVKA_SERVER_ADDR= ${ LESAVKA_SERVER_ADDR } "
" LESAVKA_SERVER_SCHEME= ${ LESAVKA_SERVER_SCHEME } "
" LESAVKA_TLS_DOMAIN= ${ LESAVKA_TLS_DOMAIN } "
" SSH_OPTS= ${ SSH_OPTS } "
" REMOTE_PULSE_CAPTURE_TOOL= ${ REMOTE_PULSE_CAPTURE_TOOL } "
" REMOTE_PULSE_VIDEO_MODE= ${ REMOTE_PULSE_VIDEO_MODE } "
" REMOTE_CAPTURE_STACK= ${ REMOTE_CAPTURE_STACK } "
" REMOTE_AUDIO_SOURCE= ${ REMOTE_AUDIO_SOURCE } "
" REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK= ${ REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK } "
" REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS= ${ LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS } "
" REMOTE_CAPTURE_READY_SETTLE_SECONDS= ${ REMOTE_CAPTURE_READY_SETTLE_SECONDS } "
"PROBE_PREBUILD=0"
" VIDEO_SIZE= ${ width } x ${ height } "
" VIDEO_FPS= ${ fps } "
"VIDEO_FORMAT=mjpeg"
" REMOTE_EXPECT_UVC_WIDTH= ${ width } "
" REMOTE_EXPECT_UVC_HEIGHT= ${ height } "
" REMOTE_EXPECT_UVC_FPS= ${ fps } "
" LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US= ${ audio_delay_us } "
" LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US= ${ video_delay_us } "
"LESAVKA_OUTPUT_DELAY_APPLY=0"
"LESAVKA_OUTPUT_DELAY_SAVE=0"
"LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0"
" LESAVKA_OUTPUT_DELAY_MIN_PAIRS= ${ min_pairs } "
" LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS= ${ LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS } "
" LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS= ${ LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS } "
" LESAVKA_OUTPUT_DELAY_MAX_STEP_US= ${ LESAVKA_SERVER_RC_TUNE_MAX_STEP_US } "
" LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS= ${ LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS } "
" LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS= ${ LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS } "
" LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS= ${ LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS } "
" LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS= ${ min_pairs } "
" PROBE_EVENT_WIDTH_CODES= ${ PROBE_EVENT_WIDTH_CODES } "
" PROBE_CONDITIONING_SECONDS= ${ conditioning_seconds } "
" PROBE_CONDITIONING_WARMUP_SECONDS= ${ LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS } "
" PROBE_CONDITIONING_GAP_SECONDS= ${ LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS } "
" PROBE_CONDITIONING_EVENT_WIDTH_CODES= ${ LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES } "
" PROBE_DURATION_SECONDS= ${ probe_duration_seconds } "
" PROBE_WARMUP_SECONDS= ${ probe_warmup_seconds } "
" ANALYSIS_TIMELINE_WINDOW= ${ analysis_timeline_window } "
" LOCAL_OUTPUT_DIR= ${ output_dir } "
)
2026-05-04 14:35:33 -03:00
set +e
2026-05-06 00:29:00 -03:00
if [ [ " ${ LESAVKA_SERVER_RC_VERBOSE_PROBES } " = = "0" ] ] ; then
env " ${ probe_env [@] } " " ${ SCRIPT_DIR } /run_upstream_av_sync.sh " >" ${ log_path } " 2>& 1
RUN_MODE_PROBE_STATUS = $?
else
env " ${ probe_env [@] } " " ${ SCRIPT_DIR } /run_upstream_av_sync.sh " 2>& 1 | tee " ${ log_path } "
RUN_MODE_PROBE_STATUS = ${ PIPESTATUS [0] }
fi
2026-05-04 14:35:33 -03:00
set -e
}
write_tune_candidate_env( ) {
local mode_result = $1
local output_env = $2
python3 - <<'PY' \
" ${ mode_result } " \
" ${ output_env } " \
" ${ LESAVKA_SERVER_RC_TUNE_MIN_PAIRS } " \
2026-05-04 14:50:47 -03:00
" ${ LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS } " \
2026-05-04 14:35:33 -03:00
" ${ LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS } " \
2026-05-04 14:50:47 -03:00
" ${ LESAVKA_SERVER_RC_TUNE_MAX_STEP_US } " \
2026-05-04 14:35:33 -03:00
" ${ LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US } "
import json
import math
import pathlib
import shlex
import sys
2026-05-04 14:50:47 -03:00
(
result_path,
output_env_path,
min_pairs_raw,
max_abs_skew_raw,
max_drift_raw,
max_step_raw,
min_change_raw,
) = sys.argv[ 1:8]
2026-05-04 14:35:33 -03:00
def env_line( key, value) :
return f"{key}={shlex.quote(str(value))}\n"
def as_int( value, default = 0) :
try:
return int( str( value) .strip( ) )
except Exception:
return default
def as_float( value, default = 0.0) :
try:
result = float( str( value) .strip( ) )
except Exception:
return default
return result if math.isfinite( result) else default
result = json.loads( pathlib.Path( result_path) .read_text( ) )
calibration = result.get( "output_delay_calibration" ) or { }
sync = result.get( "sync" ) or { }
2026-05-04 15:52:51 -03:00
min_pairs = max( 1, as_int( min_pairs_raw, 13) )
2026-05-04 14:50:47 -03:00
max_abs_skew_ms = max( 1.0, as_float( max_abs_skew_raw, 1000.0) )
2026-05-04 14:35:33 -03:00
max_drift_ms = max( 0.0, as_float( max_drift_raw, 80.0) )
2026-05-04 14:50:47 -03:00
max_step_us = max( 1, as_int( max_step_raw, 500_000) )
2026-05-04 14:35:33 -03:00
min_change_us = max( 0, as_int( min_change_raw, 5_000) )
current_audio = as_int( result.get( "audio_delay_us" ) , 0)
current_video = as_int( result.get( "video_delay_us" ) , 0)
target_audio = as_int( calibration.get( "audio_target_offset_us" ) , current_audio)
target_video = as_int( calibration.get( "video_target_offset_us" ) , current_video)
paired = as_int( calibration.get( "paired_event_count" ) , as_int( sync.get( "paired_event_count" ) , 0) )
drift_ms = as_float( calibration.get( "drift_ms" ) , as_float( sync.get( "drift_ms" ) , 0.0) )
2026-05-04 14:50:47 -03:00
max_abs_skew_ms_observed = as_float(
calibration.get( "max_abs_skew_ms" ) ,
as_float( sync.get( "p95_abs_skew_ms" ) , 0.0) ,
)
2026-05-04 14:35:33 -03:00
delta_audio = target_audio - current_audio
delta_video = target_video - current_video
2026-05-04 14:50:47 -03:00
raw_delta_us = as_int(
calibration.get( "raw_device_delta_us" ) ,
delta_video if abs( delta_video) >= abs( delta_audio) else -delta_audio,
)
2026-05-04 14:35:33 -03:00
reasons = [ ]
if result.get( "run_status" ) != 0:
reasons.append( f"seed probe exited {result.get('run_status')}" )
if paired < min_pairs:
reasons.append( f"paired_event_count {paired} < {min_pairs}" )
2026-05-04 14:50:47 -03:00
if max_abs_skew_ms_observed > max_abs_skew_ms:
reasons.append(
f"max_abs_skew_ms {max_abs_skew_ms_observed:.1f} > {max_abs_skew_ms:.1f}"
)
2026-05-04 14:35:33 -03:00
if abs( drift_ms) > max_drift_ms:
reasons.append( f"abs(drift_ms) {abs(drift_ms):.1f} > {max_drift_ms:.1f}" )
2026-05-04 14:50:47 -03:00
if abs( raw_delta_us) > max_step_us:
reasons.append( f"raw delay correction {raw_delta_us:+d}us exceeds {max_step_us}us" )
2026-05-04 14:35:33 -03:00
if target_audio < 0 or target_video < 0:
reasons.append(
f"direct confirmation delays must be non-negative, got audio={target_audio} video={target_video}"
)
if abs( delta_audio) < min_change_us and abs( delta_video) < min_change_us:
reasons.append(
f"target change audio={delta_audio:+d}us video={delta_video:+d}us is below {min_change_us}us"
)
ready = not reasons
values = {
"tune_ready" : str( ready) .lower( ) ,
"tune_reason" : "ready" if ready else "; " .join( reasons) ,
"tune_audio_delay_us" : target_audio,
"tune_video_delay_us" : target_video,
"tune_audio_delta_us" : delta_audio,
"tune_video_delta_us" : delta_video,
"tune_paired_event_count" : paired,
"tune_drift_ms" : f"{drift_ms:.3f}" ,
2026-05-04 14:50:47 -03:00
"tune_max_abs_skew_ms" : f"{max_abs_skew_ms_observed:.3f}" ,
"tune_raw_delta_us" : raw_delta_us,
2026-05-04 14:35:33 -03:00
}
with pathlib.Path( output_env_path) .open( "w" ) as handle:
for key, value in values.items( ) :
handle.write( env_line( key, value) )
PY
}
2026-05-05 19:33:56 -03:00
signal_readiness_passed( ) {
local artifact_dir = $1
local min_pairs = $2
python3 - <<'PY' " ${ artifact_dir } " " ${ min_pairs } "
import json
import pathlib
import sys
artifact_dir = pathlib.Path( sys.argv[ 1] )
min_pairs = int( sys.argv[ 2] )
def load_json( path) :
try:
return json.loads( path.read_text( ) )
except Exception:
return { }
report = load_json( artifact_dir / "report.json" )
paired = int( report.get( "paired_event_count" ) or 0)
video = int( report.get( "video_event_count" ) or 0)
audio = int( report.get( "audio_event_count" ) or 0)
verdict = report.get( "verdict" ) or { }
passed = paired >= min_pairs and video >= min_pairs and audio >= min_pairs and verdict.get( "passed" ) is True
if not passed:
reason = (
f"signal readiness needs at least {min_pairs} paired coded events with sync pass; "
f"saw paired={paired}, video={video}, audio={audio}, sync={verdict.get('status', 'unknown')}"
)
print( reason)
raise SystemExit( 0 if passed else 1)
PY
}
2026-05-05 20:27:37 -03:00
write_signal_readiness_attempt_result( ) {
local attempt = $1
local artifact_dir = $2
local run_status = $3
local run_log = $4
local min_pairs = $5
local output_json = $6
python3 - <<'PY' \
" ${ attempt } " \
" ${ artifact_dir } " \
" ${ run_status } " \
" ${ run_log } " \
" ${ min_pairs } " \
" ${ output_json } "
import json
import pathlib
import re
import sys
(
attempt_raw,
artifact_dir_raw,
run_status_raw,
run_log_raw,
min_pairs_raw,
output_json_raw,
) = sys.argv[ 1:]
def as_int( value, default = 0) :
try:
return int( str( value) .strip( ) )
except Exception:
return default
def as_float( value, default = 0.0) :
try:
return float( str( value) .strip( ) )
except Exception:
return default
def load_json( path) :
try:
return json.loads( path.read_text( ) )
except Exception:
return { }
def read_text( path) :
try:
return path.read_text( errors = "replace" )
except Exception:
return ""
attempt = as_int( attempt_raw)
run_status = as_int( run_status_raw)
min_pairs = as_int( min_pairs_raw, 1)
run_log = pathlib.Path( run_log_raw)
artifact_dir = pathlib.Path( artifact_dir_raw) if artifact_dir_raw else None
if artifact_dir is None or not artifact_dir.exists( ) :
match = re.findall( r" ^artifact_dir:\s*(.+) $" , read_text( run_log) , flags = re.MULTILINE)
if match:
artifact_dir = pathlib.Path( match[ -1] .strip( ) )
if artifact_dir is None:
artifact_dir = pathlib.Path( "__missing_signal_readiness_artifact__" )
report = load_json( artifact_dir / "report.json" )
timeline = load_json( artifact_dir / "server-output-timeline.json" )
error_log = artifact_dir / "analysis-error.log"
error_text = read_text( error_log)
2026-05-05 21:01:05 -03:00
if not error_text:
error_log = artifact_dir / "analysis-error.log.tmp"
error_text = read_text( error_log)
2026-05-05 20:27:37 -03:00
if not error_text:
error_text = read_text( run_log)
events = timeline.get( "events" ) or [ ]
server_event_count = len( events)
server_video_handoff_count = sum(
1
for event in events
if event.get( "video_feed_unix_ns" ) is not None
or event.get( "video_feed_monotonic_us" ) is not None
)
server_audio_handoff_count = sum(
1
for event in events
if event.get( "audio_push_unix_ns" ) is not None
or event.get( "audio_push_monotonic_us" ) is not None
)
verdict = report.get( "verdict" ) or { }
signature_coverage = report.get( "signature_coverage" ) or { }
video_events = as_int( report.get( "video_event_count" ) )
audio_events = as_int( report.get( "audio_event_count" ) )
paired_events = as_int( report.get( "paired_event_count" ) )
coded_pairs = as_int( signature_coverage.get( "paired_event_count" ) , paired_events)
expected_pairs = as_int( signature_coverage.get( "expected_event_count" ) )
raw_activity_delta_ms = as_float( report.get( "activity_start_delta_ms" ) )
analyzer_failure = ""
failure_match = re.search(
r"coded pulse common window removed one stream entirely;[^\n]*"
r"\(video=(?P<video>\d+)\s+audio=(?P<audio>\d+)\s+raw activity delta\s+"
r"(?P<delta>[+-]?\d+(?:\.\d+)?)\s+ms\)" ,
error_text,
)
if failure_match:
video_events = max( video_events, as_int( failure_match.group( "video" ) ) )
audio_events = max( audio_events, as_int( failure_match.group( "audio" ) ) )
raw_activity_delta_ms = as_float( failure_match.group( "delta" ) )
analyzer_failure = failure_match.group( 0)
2026-05-05 21:01:05 -03:00
pairs_match = re.search(
r"need at least\s+(?P<needed>\d+)\s+matching coded pulse pairs;\s+"
r"saw\s+(?P<paired>\d+);\s+raw activity delta was\s+"
r"(?P<delta>[+-]?\d+(?:\.\d+)?)\s+ms\s+"
r"\(video=(?P<video_time>\d+(?:\.\d+)?)s\s+audio=(?P<audio_time>\d+(?:\.\d+)?)s\)" ,
error_text,
)
if pairs_match:
paired_from_error = as_int( pairs_match.group( "paired" ) )
paired_events = max( paired_events, paired_from_error)
coded_pairs = max( coded_pairs, paired_from_error)
video_events = max( video_events, paired_from_error)
audio_events = max( audio_events, paired_from_error)
raw_activity_delta_ms = as_float( pairs_match.group( "delta" ) )
analyzer_failure = pairs_match.group( 0)
2026-05-05 20:27:37 -03:00
sync_passed = verdict.get( "passed" ) is True
paired_ready = paired_events >= min_pairs or coded_pairs >= min_pairs
video_ready = video_events >= min_pairs
audio_ready = audio_events >= min_pairs
passed = run_status = = 0 and paired_ready and video_ready and audio_ready and sync_passed
reasons = [ ]
if run_status != 0:
reasons.append( f"probe command exited {run_status}" )
if server_event_count = = 0:
reasons.append( "server did not report generated coded events" )
if server_video_handoff_count = = 0:
reasons.append( "server did not report video sink handoff events" )
if server_audio_handoff_count = = 0:
reasons.append( "server did not report audio sink handoff events" )
if video_events < min_pairs:
reasons.append( f"Tethys video coded events {video_events} < {min_pairs}" )
if audio_events < min_pairs:
reasons.append( f"Tethys audio coded events {audio_events} < {min_pairs}" )
if max( paired_events, coded_pairs) < min_pairs:
reasons.append( f"paired coded events {max(paired_events, coded_pairs)} < {min_pairs}" )
if report and not sync_passed:
reasons.append( f"sync verdict did not pass: {verdict.get('status', 'unknown')}" )
if analyzer_failure:
reasons.append( analyzer_failure)
if not reasons and passed:
reasons.append( "ready" )
elif not reasons:
reasons.append( "signal readiness failed" )
result = {
"schema" : "lesavka.server-rc-signal-readiness-attempt.v1" ,
"attempt" : attempt,
"passed" : passed,
"reason" : "; " .join( reasons) ,
"run_status" : run_status,
"run_log" : str( run_log) ,
"artifact_dir" : str( artifact_dir) if artifact_dir.exists( ) else "" ,
"analysis_error_log" : str( error_log) if error_log.exists( ) else "" ,
"counts" : {
"server_events" : server_event_count,
"server_video_handoffs" : server_video_handoff_count,
"server_audio_handoffs" : server_audio_handoff_count,
"video_events" : video_events,
"audio_events" : audio_events,
"paired_events" : paired_events,
"coded_pairs" : coded_pairs,
"expected_pairs" : expected_pairs,
} ,
"layers" : {
"server_generated_video" : server_event_count > 0,
"server_generated_audio" : server_event_count > 0,
"server_video_handoff" : server_video_handoff_count > 0,
"server_audio_handoff" : server_audio_handoff_count > 0,
"tethys_video_detected" : video_events > 0,
"tethys_audio_detected" : audio_events > 0,
"tethys_video_ready" : video_ready,
"tethys_audio_ready" : audio_ready,
"paired_coded_ready" : paired_ready,
"analyzer_sync_passed" : sync_passed,
} ,
"sync" : {
"status" : verdict.get( "status" , "unknown" ) ,
"passed" : sync_passed,
"p95_abs_skew_ms" : as_float( verdict.get( "p95_abs_skew_ms" ) ) ,
"median_skew_ms" : as_float( report.get( "median_skew_ms" ) ) ,
"drift_ms" : as_float( report.get( "drift_ms" ) ) ,
"raw_activity_delta_ms" : raw_activity_delta_ms,
} ,
"analyzer_failure" : analyzer_failure,
}
pathlib.Path( output_json_raw) .write_text( json.dumps( result, indent = 2, sort_keys = True) + "\n" )
print( result[ "reason" ] )
raise SystemExit( 0 if passed else 1)
PY
}
write_signal_readiness_attempts_summary( ) {
local output_json = $1
shift || true
python3 - <<'PY' " ${ output_json } " " $@ "
import json
import pathlib
import sys
output = pathlib.Path( sys.argv[ 1] )
attempts = [ ]
for raw in sys.argv[ 2:] :
path = pathlib.Path( raw)
try:
attempts.append( json.loads( path.read_text( ) ) )
except Exception:
attempts.append( {
"schema" : "lesavka.server-rc-signal-readiness-attempt.v1" ,
"attempt_json" : str( path) ,
"passed" : False,
"reason" : "attempt result could not be read" ,
} )
passed_attempt = next( ( attempt for attempt in attempts if attempt.get( "passed" ) ) , None)
summary = {
"schema" : "lesavka.server-rc-signal-readiness-summary.v1" ,
"passed" : passed_attempt is not None,
"passed_attempt" : passed_attempt.get( "attempt" ) if passed_attempt else None,
"attempt_count" : len( attempts) ,
"final_reason" : (
"ready"
if passed_attempt
else ( attempts[ -1] .get( "reason" ) if attempts else "no readiness attempts ran" )
) ,
"attempts" : attempts,
}
output.write_text( json.dumps( summary, indent = 2, sort_keys = True) + "\n" )
PY
}
2026-05-05 19:33:56 -03:00
write_signal_readiness_failure( ) {
local mode = $1
local width = $2
local height = $3
local fps = $4
local video_delay_us = $5
local audio_delay_us = $6
local readiness_status = $7
local readiness_log = $8
local readiness_artifact_dir = $9
2026-05-05 20:27:37 -03:00
local readiness_attempts_json = ${ 10 }
local readiness_reason = ${ 11 }
local output_json = ${ 12 }
2026-05-05 19:33:56 -03:00
python3 - <<'PY' \
" ${ mode } " \
" ${ width } " \
" ${ height } " \
" ${ fps } " \
" ${ video_delay_us } " \
" ${ audio_delay_us } " \
" ${ readiness_status } " \
" ${ readiness_log } " \
" ${ readiness_artifact_dir } " \
2026-05-05 20:27:37 -03:00
" ${ readiness_attempts_json } " \
2026-05-05 19:33:56 -03:00
" ${ readiness_reason } " \
" ${ output_json } "
import json
import pathlib
import sys
(
mode,
width_raw,
height_raw,
fps_raw,
video_delay_raw,
audio_delay_raw,
readiness_status_raw,
readiness_log,
readiness_artifact_dir,
2026-05-05 20:27:37 -03:00
readiness_attempts_json,
2026-05-05 19:33:56 -03:00
readiness_reason,
output_json,
) = sys.argv[ 1:]
def as_int( value, default = 0) :
try:
return int( str( value) .strip( ) )
except Exception:
return default
def load_json( path) :
try:
return json.loads( pathlib.Path( path) .read_text( ) )
except Exception:
return { }
report = load_json( pathlib.Path( readiness_artifact_dir) / "report.json" )
verdict = report.get( "verdict" ) or { }
signature_coverage = report.get( "signature_coverage" ) or { }
2026-05-05 20:27:37 -03:00
readiness_attempts = load_json( readiness_attempts_json) .get( "attempts" ) if readiness_attempts_json else None
2026-05-05 19:33:56 -03:00
result = {
"schema" : "lesavka.server-rc-mode-result.v1" ,
"mode" : mode,
"width" : as_int( width_raw) ,
"height" : as_int( height_raw) ,
"fps" : as_int( fps_raw) ,
"audio_delay_us" : as_int( audio_delay_raw) ,
"video_delay_us" : as_int( video_delay_raw) ,
"run_status" : as_int( readiness_status_raw) ,
"run_log" : readiness_log,
"artifact_dir" : readiness_artifact_dir,
"report_json" : str( pathlib.Path( readiness_artifact_dir) / "report.json" ) ,
"passed" : False,
"failure_reasons" : [ f"signal readiness did not pass: {readiness_reason}" ] ,
"signal_readiness" : {
"passed" : False,
"reason" : readiness_reason,
"artifact_dir" : readiness_artifact_dir,
"run_log" : readiness_log,
2026-05-05 20:27:37 -03:00
"attempts_json" : readiness_attempts_json,
"attempts" : readiness_attempts or [ ] ,
2026-05-05 19:33:56 -03:00
} ,
"sync" : {
"passed" : verdict.get( "passed" ) is True,
"status" : verdict.get( "status" , "unknown" ) ,
"reason" : verdict.get( "reason" , "" ) ,
"p95_abs_skew_ms" : float( verdict.get( "p95_abs_skew_ms" ) or 0.0) ,
"median_skew_ms" : float( report.get( "median_skew_ms" ) or 0.0) ,
"drift_ms" : float( report.get( "drift_ms" ) or 0.0) ,
"activity_start_delta_ms" : float( report.get( "activity_start_delta_ms" ) or 0.0) ,
"first_skew_ms" : float( report.get( "first_skew_ms" ) or 0.0) ,
"video_event_count" : as_int( report.get( "video_event_count" ) ) ,
"audio_event_count" : as_int( report.get( "audio_event_count" ) ) ,
"paired_event_count" : as_int( report.get( "paired_event_count" ) ) ,
"signature_expected_event_count" : as_int( signature_coverage.get( "expected_event_count" ) ) ,
"signature_paired_event_count" : as_int( signature_coverage.get( "paired_event_count" ) ) ,
"signature_missing_event_ids" : signature_coverage.get( "missing_event_ids" ) or [ ] ,
"signature_missing_codes" : signature_coverage.get( "missing_codes" ) or [ ] ,
"signature_unknown_pair_identity_count" : as_int( signature_coverage.get( "unknown_pair_identity_count" ) ) ,
} ,
"freshness" : {
"status" : "unknown" ,
"reason" : "measured probe skipped because signal readiness failed" ,
"worst_event_age_with_uncertainty_ms" : 0.0,
} ,
"smoothness" : { } ,
"output_delay_calibration" : { } ,
}
pathlib.Path( output_json) .write_text( json.dumps( result, indent = 2, sort_keys = True) + "\n" )
PY
}
2026-05-04 12:48:17 -03:00
discover_local_webcam_modes( ) {
if ! command -v python3 >/dev/null 2>& 1; then
printf 'LESAVKA_SERVER_RC_MODES=auto requires python3 for local webcam mode discovery.\n' >& 2
exit 64
fi
if ! command -v v4l2-ctl >/dev/null 2>& 1; then
printf 'LESAVKA_SERVER_RC_MODES=auto requires v4l2-ctl for local webcam mode discovery.\n' >& 2
exit 64
fi
python3 - <<'PY' \
" ${ LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES } " \
" ${ LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS } " \
" ${ LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX } " \
" ${ LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX } "
import glob
import re
import subprocess
import sys
sizes_raw, fps_raw, include_raw, exclude_raw = sys.argv[ 1:5]
def parse_sizes( raw) :
sizes = set( )
for entry in raw.split( "," ) :
entry = entry.strip( )
if not entry:
continue
match = re.fullmatch( r"(\d+)x(\d+)" , entry)
if not match:
raise SystemExit( f"invalid discovery size {entry!r}; expected WIDTHxHEIGHT" )
sizes.add( ( int( match.group( 1) ) , int( match.group( 2) ) ) )
return sizes
def parse_fps( raw) :
values = set( )
for entry in raw.split( "," ) :
entry = entry.strip( )
if not entry:
continue
if not entry.isdigit( ) :
raise SystemExit( f"invalid discovery fps {entry!r}; expected integer FPS" )
values.add( int( entry) )
return values
allowed_sizes = parse_sizes( sizes_raw)
allowed_fps = parse_fps( fps_raw)
include = re.compile( include_raw, re.IGNORECASE) if include_raw else None
exclude = re.compile( exclude_raw, re.IGNORECASE) if exclude_raw else None
all_modes = set( )
camera_modes = [ ]
for dev in sorted( glob.glob( "/dev/video*" ) ) :
try:
info = subprocess.run(
[ "v4l2-ctl" , "-d" , dev, "--info" ] ,
check = True,
stdout = subprocess.PIPE,
stderr = subprocess.DEVNULL,
text = True,
) .stdout
formats = subprocess.run(
[ "v4l2-ctl" , "-d" , dev, "--list-formats-ext" ] ,
check = True,
stdout = subprocess.PIPE,
stderr = subprocess.DEVNULL,
text = True,
) .stdout
except Exception:
continue
card = dev
for line in info.splitlines( ) :
if "Card type" in line:
card = line.split( ":" , 1) [ 1] .strip( )
break
label = f"{card} {dev}"
if exclude and exclude.search( label) :
continue
if include and not include.search( label) :
continue
current_size = None
modes = set( )
for line in formats.splitlines( ) :
size_match = re.search( r"Size:\s+Discrete\s+(\d+)x(\d+)" , line)
if size_match:
current_size = ( int( size_match.group( 1) ) , int( size_match.group( 2) ) )
continue
fps_match = re.search( r"Interval:\s+Discrete\s+[^()]+\((\d+(?:\.\d+)?) fps\)" , line)
if not fps_match or current_size not in allowed_sizes:
continue
fps_float = float( fps_match.group( 1) )
fps = int( round( fps_float) )
if abs( fps_float - fps) > 0.05 or fps not in allowed_fps:
continue
modes.add( ( current_size[ 0] , current_size[ 1] , fps) )
if modes:
all_modes.update( modes)
camera_modes.append( ( label, modes) )
if not all_modes:
print(
"no matching local webcam modes found; "
f"sizes={sizes_raw} fps={fps_raw} include={include_raw!r} exclude={exclude_raw!r}" ,
file = sys.stderr,
)
raise SystemExit( 64)
for label, modes in camera_modes:
rendered = "," .join( f"{w}x{h}@{fps}" for w, h, fps in sorted( modes) )
print( f" -> local webcam {label}: {rendered}" , file = sys.stderr)
print( "," .join( f"{w}x{h}@{fps}" for w, h, fps in sorted( all_modes) ) , end = "" )
PY
}
clear_remote_sudo_password( ) {
LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD = ""
}
trap clear_remote_sudo_password EXIT
ensure_remote_sudo_password( ) {
[ [ " ${ LESAVKA_SERVER_RC_RECONFIGURE } " != "0" ] ] || return 0
[ [ -z " ${ LESAVKA_SERVER_RC_RECONFIGURE_COMMAND } " ] ] || return 0
[ [ -n " ${ LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD } " ] ] && return 0
if [ [ " ${ LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY } " = = "0" ] ] ; then
printf 'remote sudo password is required for automatic reconfigure; set LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD or leave LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY=1.\n' >& 2
exit 64
fi
if [ [ ! -t 0 ] ] ; then
printf 'remote sudo password is required, but stdin is not a terminal.\n' >& 2
printf 'Re-run from a terminal or set LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD in the environment.\n' >& 2
exit 64
fi
printf 'Theia sudo password for %s: ' " ${ LESAVKA_SERVER_HOST } " >& 2
IFS = read -r -s LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD
printf '\n' >& 2
}
run_remote_root_script( ) {
local description = $1
shift
local script_file remote_wrapper ssh_cmd status
script_file = " $( mktemp) "
cat >" ${ script_file } "
remote_wrapper = ' set -euo pipefail
read -r __lesavka_sudo_password
__lesavka_script = $( mktemp)
cleanup( ) { rm -f " $__lesavka_script " ; }
trap cleanup EXIT
cat >" $__lesavka_script "
printf "%s\n" " $__lesavka_sudo_password " | sudo -S -p "" -v
printf "%s\n" " $__lesavka_sudo_password " | sudo -S -p "" bash " $__lesavka_script " " $@ "
'
printf -v ssh_cmd 'bash -c %q _' " ${ remote_wrapper } "
for arg in " $@ " ; do
printf -v ssh_cmd '%s %q' " ${ ssh_cmd } " " ${ arg } "
done
set +e
{
printf '%s\n' " ${ LESAVKA_SERVER_RC_REMOTE_SUDO_PASSWORD } "
cat " ${ script_file } "
} | ssh ${ SSH_OPTS } " ${ LESAVKA_SERVER_HOST } " " ${ ssh_cmd } "
status = $?
set -e
rm -f " ${ script_file } "
if [ [ " ${ status } " -ne 0 ] ] ; then
printf '%s failed on %s with exit %s\n' " ${ description } " " ${ LESAVKA_SERVER_HOST } " " ${ status } " >& 2
fi
return " ${ status } "
}
prime_remote_sudo( ) {
[ [ " ${ LESAVKA_SERVER_RC_RECONFIGURE } " != "0" ] ] || return 0
[ [ -z " ${ LESAVKA_SERVER_RC_RECONFIGURE_COMMAND } " ] ] || return 0
ensure_remote_sudo_password
echo " ==> priming remote sudo on ${ LESAVKA_SERVER_HOST } "
run_remote_root_script "remote sudo prime" <<'REMOTE_SUDO_PR IME'
set -euo pipefail
true
REMOTE_SUDO_PRIME
}
2026-05-04 20:32:21 -03:00
sleep_start_delay( ) {
[ [ " ${ LESAVKA_SERVER_RC_START_DELAY_SECONDS } " != "0" ] ] || return 0
if [ [ ! " ${ LESAVKA_SERVER_RC_START_DELAY_SECONDS } " = ~ ^[ 0-9] +( [ .] [ 0-9] +) ?$ ] ] ; then
printf 'LESAVKA_SERVER_RC_START_DELAY_SECONDS must be a non-negative number, got %s\n' " ${ LESAVKA_SERVER_RC_START_DELAY_SECONDS } " >& 2
exit 64
fi
echo " ==> delaying server-to-RC matrix start for ${ LESAVKA_SERVER_RC_START_DELAY_SECONDS } s "
echo " ↪ remote sudo has already been primed; sleeping before prebuild/reconfigure/capture"
sleep " ${ LESAVKA_SERVER_RC_START_DELAY_SECONDS } "
}
2026-05-04 12:48:17 -03:00
prebuild_probe_tools( ) {
[ [ " ${ LESAVKA_SERVER_RC_PROBE_PREBUILD } " != "0" ] ] || return 0
echo "==> prebuilding relay control/analyzer once for the mode matrix"
(
cd " ${ REPO_ROOT } "
cargo build -p lesavka_client --bin lesavka-sync-analyze --bin lesavka-relayctl
)
2026-05-04 00:47:21 -03:00
}
reconfigure_server_mode( ) {
local mode = $1
local width = $2
local height = $3
local fps = $4
2026-05-06 03:59:20 -03:00
local audio_delay_us = ${ 5 :- $( lookup_audio_delay_us " ${ mode } " ) }
local video_delay_us = ${ 6 :- $( lookup_video_delay_us " ${ mode } " ) }
2026-05-04 00:47:21 -03:00
[ [ " ${ LESAVKA_SERVER_RC_RECONFIGURE } " != "0" ] ] || return 0
echo " ==> reconfiguring ${ LESAVKA_SERVER_HOST } UVC gadget for ${ mode } "
if [ [ -n " ${ LESAVKA_SERVER_RC_RECONFIGURE_COMMAND } " ] ] ; then
LESAVKA_MODE = " ${ mode } " \
LESAVKA_UVC_WIDTH = " ${ width } " \
LESAVKA_UVC_HEIGHT = " ${ height } " \
LESAVKA_UVC_FPS = " ${ fps } " \
2026-05-06 03:59:20 -03:00
LESAVKA_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US = " ${ audio_delay_us } " \
LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US = " ${ video_delay_us } " \
2026-05-04 00:47:21 -03:00
bash -c " ${ LESAVKA_SERVER_RC_RECONFIGURE_COMMAND } "
return 0
fi
2026-05-04 12:48:17 -03:00
if [ [ " ${ LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY } " != "runtime" ] ] ; then
printf 'unsupported LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY=%s\n' " ${ LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY } " >& 2
printf 'Use runtime for fast mode changes, or LESAVKA_SERVER_RC_RECONFIGURE_COMMAND for a custom install/reconfigure command.\n' >& 2
return 64
fi
local interval
interval = $(( 10000000 / fps))
run_remote_root_script " runtime UVC reconfigure for ${ mode } " \
" ${ mode } " \
" ${ width } " \
" ${ height } " \
" ${ fps } " \
" ${ interval } " \
" ${ LESAVKA_SERVER_RC_RECONFIGURE_CODEC } " \
" ${ LESAVKA_SERVER_RC_ALLOW_GADGET_RESET } " \
" ${ LESAVKA_SERVER_RC_FORCE_GADGET_REBUILD } " \
" ${ LESAVKA_SERVER_RC_RECONFIGURE_SETTLE_SECONDS } " \
2026-05-06 03:59:20 -03:00
" ${ LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE } " \
" ${ audio_delay_us } " \
" ${ video_delay_us } " \
" ${ LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US } " \
" ${ LESAVKA_SERVER_RC_MODE_DELAYS_US } " <<'REMOTE_RE CONFIGURE'
2026-05-04 00:47:21 -03:00
set -euo pipefail
2026-05-04 12:48:17 -03:00
mode = $1
width = $2
height = $3
fps = $4
interval = $5
codec = $6
allow_gadget_reset = $7
force_gadget_rebuild = $8
settle_seconds = $9
verbose = ${ 10 }
2026-05-06 03:59:20 -03:00
audio_delay_us = ${ 11 }
video_delay_us = ${ 12 }
audio_delay_map = ${ 13 }
video_delay_map = ${ 14 }
2026-05-04 12:48:17 -03:00
set_env_value( ) {
local file = $1
local key = $2
local value = $3
local tmp
tmp = $( mktemp)
if [ [ -f " ${ file } " ] ] ; then
awk -v key = " ${ key } " -v value = " ${ value } " '
BEGIN { wrote = 0 }
$0 ~ "^" key "=" {
print key "=" value
wrote = 1
next
}
{ print }
END {
if ( !wrote) {
print key "=" value
}
}
' " ${ file } " >" ${ tmp } "
else
printf '%s=%s\n' " ${ key } " " ${ value } " >" ${ tmp } "
fi
install -m 0644 " ${ tmp } " " ${ file } "
rm -f " ${ tmp } "
}
if [ [ ! -x /usr/local/bin/lesavka-core.sh ] ] ; then
printf 'missing /usr/local/bin/lesavka-core.sh; run the server installer once before using fast runtime reconfigure.\n' >& 2
exit 65
2026-05-04 01:01:08 -03:00
fi
2026-05-04 12:48:17 -03:00
if [ [ ! -x /usr/local/bin/lesavka-server || ! -x /usr/local/bin/lesavka-uvc ] ] ; then
printf 'missing installed Lesavka binaries; run the server installer once before using fast runtime reconfigure.\n' >& 2
2026-05-04 01:01:08 -03:00
exit 65
fi
2026-05-04 12:48:17 -03:00
install -d -m 0755 /etc/lesavka
touch /etc/lesavka/server.env /etc/lesavka/uvc.env
set_env_value /etc/lesavka/server.env LESAVKA_CAM_OUTPUT uvc
set_env_value /etc/lesavka/server.env LESAVKA_UVC_CODEC " ${ codec } "
set_env_value /etc/lesavka/server.env LESAVKA_UVC_WIDTH " ${ width } "
set_env_value /etc/lesavka/server.env LESAVKA_UVC_HEIGHT " ${ height } "
set_env_value /etc/lesavka/server.env LESAVKA_UVC_FPS " ${ fps } "
set_env_value /etc/lesavka/server.env LESAVKA_UVC_INTERVAL " ${ interval } "
2026-05-06 03:59:20 -03:00
set_env_value /etc/lesavka/server.env LESAVKA_UPSTREAM_AUDIO_PLAYOUT_MODE_OFFSETS_US " ${ audio_delay_map } "
set_env_value /etc/lesavka/server.env LESAVKA_UPSTREAM_VIDEO_PLAYOUT_MODE_OFFSETS_US " ${ video_delay_map } "
set_env_value /etc/lesavka/server.env LESAVKA_UPSTREAM_AUDIO_PLAYOUT_OFFSET_US " ${ audio_delay_us } "
set_env_value /etc/lesavka/server.env LESAVKA_UPSTREAM_VIDEO_PLAYOUT_OFFSET_US " ${ video_delay_us } "
2026-05-04 12:48:17 -03:00
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_CODEC " ${ codec } "
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_WIDTH " ${ width } "
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_HEIGHT " ${ height } "
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_FPS " ${ fps } "
set_env_value /etc/lesavka/uvc.env LESAVKA_UVC_INTERVAL " ${ interval } "
printf ' ↪ fast runtime env updated: CAM_OUTPUT=uvc UVC_MODE=%s codec=%s\n' " ${ mode } " " ${ codec } "
systemctl daemon-reload
systemctl stop lesavka-server >/dev/null 2>& 1 || true
systemctl stop lesavka-uvc >/dev/null 2>& 1 || true
systemctl reset-failed lesavka-core lesavka-uvc lesavka-server >/dev/null 2>& 1 || true
if [ [ " ${ allow_gadget_reset } " != "0" && " ${ force_gadget_rebuild } " != "0" ] ] ; then
printf ' ↪ cycling UVC gadget descriptors for %s\n' " ${ mode } "
mode_log_id = $( printf '%s' " ${ mode } " | tr -c '[:alnum:]_.-' '_' )
core_log = " /tmp/lesavka-core-reconfigure- ${ mode_log_id } .log "
core_cmd = (
env
LESAVKA_ALLOW_GADGET_RESET = 1 \
LESAVKA_FORCE_GADGET_REBUILD = 1 \
LESAVKA_ATTACH_WRITE_UDC = 1 \
LESAVKA_DETACH_CLEAR_UDC = 1 \
LESAVKA_UVC_FALLBACK = 0 \
LESAVKA_UVC_CODEC = " ${ codec } " \
LESAVKA_UVC_WIDTH = " ${ width } " \
LESAVKA_UVC_HEIGHT = " ${ height } " \
LESAVKA_UVC_FPS = " ${ fps } " \
LESAVKA_UVC_INTERVAL = " ${ interval } " \
/usr/local/bin/lesavka-core.sh
)
if [ [ " ${ verbose } " != "0" ] ] ; then
" ${ core_cmd [@] } "
else
if " ${ core_cmd [@] } " >" ${ core_log } " 2>& 1; then
printf ' ↪ lesavka-core reconfigure log: %s\n' " ${ core_log } "
else
rc = $?
printf 'lesavka-core reconfigure failed; tail of %s follows\n' " ${ core_log } " >& 2
tail -n 80 " ${ core_log } " >& 2 || true
exit " ${ rc } "
fi
2026-05-04 01:12:20 -03:00
fi
2026-05-04 12:48:17 -03:00
elif [ [ " ${ allow_gadget_reset } " != "0" ] ] ; then
printf ' ↪ gadget reset allowed, but force rebuild disabled; refreshing services only\n'
2026-05-04 01:12:20 -03:00
else
2026-05-04 12:48:17 -03:00
printf ' ↪ preserving attached UVC gadget; descriptors may remain on the previous mode\n'
2026-05-04 01:12:20 -03:00
fi
2026-05-04 01:22:25 -03:00
2026-05-04 12:48:17 -03:00
systemctl start lesavka-uvc
systemctl restart lesavka-server
sleep " ${ settle_seconds } "
systemctl is-active lesavka-core lesavka-uvc lesavka-server >/dev/null
printf ' ↪ services active after %s reconfigure\n' " ${ mode } "
REMOTE_RECONFIGURE
}
wait_tethys_media_ready( ) {
local mode = $1
local width = $2
local height = $3
local fps = $4
[ [ " ${ LESAVKA_SERVER_RC_WAIT_TETHYS_READY } " != "0" ] ] || return 0
echo " ==> waiting for Tethys media endpoints for ${ mode } "
ssh ${ SSH_OPTS } " ${ TETHYS_HOST } " bash -s -- \
" ${ mode } " \
2026-05-04 01:22:25 -03:00
" ${ width } " \
" ${ height } " \
" ${ fps } " \
2026-05-04 12:48:17 -03:00
" ${ LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS } " \
" ${ LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS } " \
" ${ REMOTE_CAPTURE_STACK } " \
" ${ REMOTE_AUDIO_SOURCE } " <<'REMOTE_TETHYS_RE ADY'
set -euo pipefail
mode = $1
width = $2
height = $3
fps = $4
timeout_seconds = $5
settle_seconds = $6
capture_stack = $7
audio_source = $8
sleep " ${ settle_seconds } "
find_lesavka_video_device( ) {
if [ [ -d /dev/v4l/by-id ] ] ; then
while IFS = read -r path; do
[ [ -e " ${ path } " ] ] || continue
printf '%s\n' " ${ path } "
return 0
done < <( find /dev/v4l/by-id -maxdepth 1 -type l -name '*Lesavka*video-index0' 2>/dev/null | sort)
fi
if command -v v4l2-ctl >/dev/null 2>& 1; then
v4l2-ctl --list-devices 2>/dev/null \
| awk '
BEGIN { want = 0 }
/Lesavka Composite| Lesavka.*UVC/ { want = 1; next }
/^[ ^ \t ] / { want = 0 }
want && /^[ \t ] +\/ dev\/ video[ 0-9] +/ {
gsub( /^[ \t ] +/, "" , $0 )
print
exit
}
'
fi
}
video_ready( ) {
local dev = $1
[ [ -n " ${ dev } " ] ] || return 1
[ [ -e " ${ dev } " ] ] || return 1
if ! command -v v4l2-ctl >/dev/null 2>& 1; then
return 0
fi
local listing
listing = " $( v4l2-ctl -d " ${ dev } " --list-formats-ext 2>/dev/null || true ) "
grep -q " Size: Discrete ${ width } x ${ height } " <<< " ${ listing } " || return 1
grep -Eq " (^|[^0-9]) ${ fps } (\\.0+)? fps " <<< " ${ listing } " || return 1
}
pulse_ready( ) {
if ! command -v pactl >/dev/null 2>& 1; then
return 1
fi
if [ [ " ${ audio_source } " = = pulse:* ] ] ; then
local requested = ${ audio_source #pulse : }
pactl list short sources 2>/dev/null | awk -v requested = " ${ requested } " '$2 == requested { found=1 } END { exit found ? 0 : 1 }'
return
fi
pactl list short sources 2>/dev/null | grep -Eq 'alsa_input\..*Lesavka_Composite|Lesavka_Composite'
}
alsa_ready( ) {
if [ [ " ${ audio_source } " = = alsa:* ] ] ; then
[ [ -e " ${ audio_source #alsa : } " ] ] || return 1
return 0
fi
command -v arecord >/dev/null 2>& 1 || return 1
arecord -l 2>/dev/null | grep -Eq 'Lesavka|UAC2_Gadget|UAC2Gadget|Composite'
}
audio_ready( ) {
case " ${ capture_stack } " in
pulse)
pulse_ready
; ;
alsa)
alsa_ready
; ;
auto)
pulse_ready || alsa_ready
; ;
pwpipe)
command -v pw-dump >/dev/null 2>& 1 && pw-dump 2>/dev/null | grep -Eq 'Lesavka_Composite|Lesavka Composite'
; ;
*)
pulse_ready || alsa_ready
; ;
esac
}
deadline = $(( SECONDS + timeout_seconds))
last_video = 'none'
last_audio = 'none'
while ( ( SECONDS <= deadline ) ) ; do
video_dev = " $( find_lesavka_video_device || true ) "
last_video = " ${ video_dev :- none } "
if [ [ -n " ${ video_dev } " ] ] && video_ready " ${ video_dev } " ; then
if audio_ready; then
printf ' ↪ Tethys media ready: video=%s mode=%s audio_stack=%s\n' " ${ video_dev } " " ${ mode } " " ${ capture_stack } "
exit 0
fi
last_audio = 'not-ready'
fi
sleep 0.5
done
printf 'timed out waiting for Tethys Lesavka media endpoints for %s after %ss\n' " ${ mode } " " ${ timeout_seconds } " >& 2
printf 'last video candidate: %s\n' " ${ last_video } " >& 2
printf 'last audio candidate: %s\n' " ${ last_audio } " >& 2
if command -v v4l2-ctl >/dev/null 2>& 1; then
v4l2-ctl --list-devices >& 2 || true
if [ [ " ${ last_video } " != "none" ] ] ; then
v4l2-ctl -d " ${ last_video } " --list-formats-ext >& 2 || true
fi
fi
if command -v pactl >/dev/null 2>& 1; then
pactl list short sources | grep -iE 'lesavka|uac|composite' >& 2 || true
fi
if command -v arecord >/dev/null 2>& 1; then
arecord -l >& 2 || true
fi
exit 70
REMOTE_TETHYS_READY
2026-05-04 00:47:21 -03:00
}
write_mode_result( ) {
local mode = $1
local width = $2
local height = $3
local fps = $4
local video_delay_us = $5
2026-05-04 12:48:17 -03:00
local audio_delay_us = $6
local run_status = $7
local run_log = $8
local artifact_dir = $9
local output_json = ${ 10 }
2026-05-04 00:47:21 -03:00
python3 - <<'PY' \
" ${ mode } " \
" ${ width } " \
" ${ height } " \
" ${ fps } " \
" ${ video_delay_us } " \
2026-05-04 12:48:17 -03:00
" ${ audio_delay_us } " \
2026-05-04 00:47:21 -03:00
" ${ run_status } " \
" ${ run_log } " \
" ${ artifact_dir } " \
" ${ output_json } " \
" ${ LESAVKA_SERVER_RC_REQUIRE_SYNC_PASS } " \
" ${ LESAVKA_SERVER_RC_REQUIRE_FRESHNESS_PASS } " \
" ${ LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS } " \
" ${ LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS } " \
" ${ LESAVKA_SERVER_RC_MAX_VIDEO_P95_JITTER_MS } " \
" ${ LESAVKA_SERVER_RC_MAX_AUDIO_P95_JITTER_MS } " \
" ${ LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES } " \
" ${ LESAVKA_SERVER_RC_MAX_VIDEO_UNDECODABLE_FRAMES } " \
" ${ LESAVKA_SERVER_RC_MAX_VIDEO_DUPLICATE_FRAMES } " \
2026-05-05 19:33:56 -03:00
" ${ LESAVKA_SERVER_RC_MAX_AUDIO_LOW_RMS_WINDOWS } " \
" ${ LESAVKA_SERVER_RC_MIN_CODED_PAIRS } " \
" ${ LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS } " \
" ${ LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS } "
2026-05-04 00:47:21 -03:00
import json
import math
import pathlib
import sys
(
mode,
width_raw,
height_raw,
fps_raw,
video_delay_raw,
2026-05-04 12:48:17 -03:00
audio_delay_raw,
2026-05-04 00:47:21 -03:00
run_status_raw,
run_log,
artifact_dir_raw,
output_json,
require_sync_raw,
require_freshness_raw,
max_video_hiccups_raw,
max_audio_hiccups_raw,
max_video_jitter_raw,
max_audio_jitter_raw,
max_missing_raw,
max_undecodable_raw,
max_duplicates_raw,
max_low_rms_raw,
2026-05-05 19:33:56 -03:00
min_coded_pairs_raw,
require_all_coded_pairs_raw,
require_smoothness_raw,
2026-05-04 00:47:21 -03:00
) = sys.argv[ 1:]
def as_bool( value) :
return str( value) .strip( ) .lower( ) not in { "" , "0" , "false" , "no" , "off" }
def as_int( value, default = 0) :
try:
return int( str( value) .strip( ) )
except Exception:
return default
def as_float( value, default = 0.0) :
try:
result = float( str( value) .strip( ) )
except Exception:
return default
return result if math.isfinite( result) else default
def load_json( path) :
try:
return json.loads( pathlib.Path( path) .read_text( ) )
except Exception:
return { }
def nested( mapping, *keys, default = None) :
value = mapping
for key in keys:
if not isinstance( value, dict) :
return default
value = value.get( key)
return default if value is None else value
artifact_dir = pathlib.Path( artifact_dir_raw) if artifact_dir_raw else pathlib.Path( )
report = load_json( artifact_dir / "report.json" )
correlation = load_json( artifact_dir / "output-delay-correlation.json" )
2026-05-04 12:48:17 -03:00
calibration = load_json( artifact_dir / "output-delay-calibration.json" )
2026-05-04 18:47:22 -03:00
timing_map = correlation.get( "timing_map" ) or { }
2026-05-04 00:47:21 -03:00
freshness = correlation.get( "freshness" ) or { }
2026-05-04 18:47:22 -03:00
capture_timebase = freshness.get( "capture_timebase" ) or { }
2026-05-04 00:47:21 -03:00
smoothness = correlation.get( "smoothness" ) or { }
2026-05-05 19:33:56 -03:00
event_visibility = correlation.get( "event_visibility" ) or { }
2026-05-04 00:47:21 -03:00
video = smoothness.get( "video" ) or { }
audio = smoothness.get( "audio" ) or { }
audio_cadence = audio.get( "packet_cadence" ) or { }
audio_rms = audio.get( "rms_continuity" ) or { }
verdict = report.get( "verdict" ) or { }
2026-05-04 16:58:55 -03:00
signature_coverage = report.get( "signature_coverage" ) or { }
2026-05-04 00:47:21 -03:00
sync_pass = verdict.get( "passed" ) is True
freshness_status = freshness.get( "status" , "unknown" )
freshness_pass = freshness_status = = "pass"
run_status = as_int( run_status_raw, 1)
video_hiccups = as_int( video.get( "hiccup_count" ) , 0)
audio_hiccups = as_int( audio_cadence.get( "hiccup_count" ) , 0)
video_jitter = as_float( nested( video, "jitter_stats" , "p95_jitter_ms" ) , 0.0)
audio_jitter = as_float( nested( audio_cadence, "jitter_stats" , "p95_jitter_ms" ) , 0.0)
missing = as_int( video.get( "estimated_missing_frames" ) , 0)
undecodable = as_int( video.get( "undecodable_frames" ) , 0)
duplicates = as_int( video.get( "duplicate_frames" ) , 0)
low_rms = as_int( audio_rms.get( "low_rms_window_count" ) , 0)
2026-05-04 16:58:55 -03:00
signature_expected = as_int( signature_coverage.get( "expected_event_count" ) , 0)
signature_paired = as_int( signature_coverage.get( "paired_event_count" ) , 0)
signature_unknown = as_int( signature_coverage.get( "unknown_pair_identity_count" ) , 0)
2026-05-05 19:33:56 -03:00
min_coded_pairs = as_int( min_coded_pairs_raw, 0)
require_all_coded_pairs = as_bool( require_all_coded_pairs_raw)
require_smoothness = as_bool( require_smoothness_raw)
2026-05-04 14:50:47 -03:00
pair_confidences = [ ]
for event in report.get( "paired_events" ) or [ ] :
if not isinstance( event, dict) :
continue
try:
confidence = float( event.get( "confidence" ) )
except Exception:
continue
if math.isfinite( confidence) :
pair_confidences.append( confidence)
def median( values, default = 0.0) :
if not values:
return default
ordered = sorted( values)
mid = len( ordered) // 2
if len( ordered) % 2:
return ordered[ mid]
return ( ordered[ mid - 1] + ordered[ mid] ) / 2.0
activity_start_delta_ms = as_float( report.get( "activity_start_delta_ms" ) , 0.0)
first_skew_ms = as_float( report.get( "first_skew_ms" ) , 0.0)
activity_pair_disagreement_ms = (
activity_start_delta_ms - first_skew_ms
if as_int( report.get( "paired_event_count" ) , 0) > 0
else 0.0
)
2026-05-04 00:47:21 -03:00
reasons = [ ]
2026-05-05 19:33:56 -03:00
smoothness_warnings = [ ]
2026-05-04 00:47:21 -03:00
if run_status != 0:
reasons.append( f"probe command exited {run_status}" )
if as_bool( require_sync_raw) and not sync_pass:
reasons.append( f"sync did not pass: {verdict.get('status', 'unknown')}" )
if as_bool( require_freshness_raw) and not freshness_pass:
reasons.append( f"freshness did not pass: {freshness_status}" )
2026-05-04 18:47:22 -03:00
if capture_timebase and capture_timebase.get( "valid" ) is False:
reasons.append(
"capture timebase invalid: "
f"{capture_timebase.get('reason', capture_timebase.get('status', 'invalid'))}"
)
2026-05-04 16:58:55 -03:00
if signature_expected > 0:
2026-05-05 19:33:56 -03:00
if require_all_coded_pairs and signature_paired < signature_expected:
2026-05-04 16:58:55 -03:00
reasons.append( f"paired coded signatures {signature_paired} < expected {signature_expected}" )
2026-05-05 19:33:56 -03:00
elif min_coded_pairs > 0 and signature_paired < min_coded_pairs:
reasons.append( f"paired coded signatures {signature_paired} < required {min_coded_pairs}" )
2026-05-04 16:58:55 -03:00
if signature_unknown > 0:
reasons.append( f"paired signatures without coded identity {signature_unknown} > 0" )
elif report.get( "coded_events" ) is True:
reasons.append( "coded signature coverage unavailable" )
2026-05-04 00:47:21 -03:00
if video_hiccups > as_int( max_video_hiccups_raw) :
2026-05-05 19:33:56 -03:00
smoothness_warnings.append( f"video hiccups {video_hiccups} > {max_video_hiccups_raw}" )
2026-05-04 00:47:21 -03:00
if audio_hiccups > as_int( max_audio_hiccups_raw) :
2026-05-05 19:33:56 -03:00
smoothness_warnings.append( f"audio hiccups {audio_hiccups} > {max_audio_hiccups_raw}" )
2026-05-04 00:47:21 -03:00
if video_jitter > as_float( max_video_jitter_raw) :
2026-05-05 19:33:56 -03:00
smoothness_warnings.append( f"video p95 jitter {video_jitter:.1f}ms > {as_float(max_video_jitter_raw):.1f}ms" )
2026-05-04 00:47:21 -03:00
if audio_jitter > as_float( max_audio_jitter_raw) :
2026-05-05 19:33:56 -03:00
smoothness_warnings.append( f"audio p95 jitter {audio_jitter:.1f}ms > {as_float(max_audio_jitter_raw):.1f}ms" )
2026-05-04 00:47:21 -03:00
if missing > as_int( max_missing_raw) :
2026-05-05 19:33:56 -03:00
smoothness_warnings.append( f"estimated missing video frames {missing} > {max_missing_raw}" )
2026-05-04 00:47:21 -03:00
if undecodable > as_int( max_undecodable_raw) :
2026-05-05 19:33:56 -03:00
smoothness_warnings.append( f"undecodable video frames {undecodable} > {max_undecodable_raw}" )
2026-05-04 00:47:21 -03:00
if duplicates > as_int( max_duplicates_raw) :
2026-05-05 19:33:56 -03:00
smoothness_warnings.append( f"duplicate video frames {duplicates} > {max_duplicates_raw}" )
2026-05-04 00:47:21 -03:00
if low_rms > as_int( max_low_rms_raw) :
2026-05-05 19:33:56 -03:00
smoothness_warnings.append( f"low-RMS audio windows {low_rms} > {max_low_rms_raw}" )
if require_smoothness:
reasons.extend( smoothness_warnings)
2026-05-04 00:47:21 -03:00
artifact = {
"schema" : "lesavka.server-rc-mode-result.v1" ,
"mode" : mode,
"width" : as_int( width_raw) ,
"height" : as_int( height_raw) ,
"fps" : as_int( fps_raw) ,
2026-05-04 12:48:17 -03:00
"audio_delay_us" : as_int( audio_delay_raw) ,
2026-05-04 00:47:21 -03:00
"video_delay_us" : as_int( video_delay_raw) ,
"run_status" : run_status,
"run_log" : run_log,
"artifact_dir" : str( artifact_dir) ,
"report_json" : str( artifact_dir / "report.json" ) ,
"correlation_json" : str( artifact_dir / "output-delay-correlation.json" ) ,
2026-05-04 12:48:17 -03:00
"calibration_json" : str( artifact_dir / "output-delay-calibration.json" ) ,
2026-05-04 18:47:22 -03:00
"timing_map" : timing_map,
2026-05-04 00:47:21 -03:00
"passed" : not reasons,
"failure_reasons" : reasons,
2026-05-05 19:33:56 -03:00
"smoothness_required" : require_smoothness,
"smoothness_warnings" : smoothness_warnings,
"event_visibility" : event_visibility,
2026-05-04 00:47:21 -03:00
"sync" : {
"passed" : sync_pass,
"status" : verdict.get( "status" , "unknown" ) ,
"reason" : verdict.get( "reason" , "" ) ,
"p95_abs_skew_ms" : as_float( verdict.get( "p95_abs_skew_ms" ) , 0.0) ,
"median_skew_ms" : as_float( report.get( "median_skew_ms" ) , 0.0) ,
"drift_ms" : as_float( report.get( "drift_ms" ) , 0.0) ,
2026-05-04 14:50:47 -03:00
"activity_start_delta_ms" : activity_start_delta_ms,
"first_skew_ms" : first_skew_ms,
"activity_pair_disagreement_ms" : activity_pair_disagreement_ms,
"video_event_count" : as_int( report.get( "video_event_count" ) , 0) ,
"audio_event_count" : as_int( report.get( "audio_event_count" ) , 0) ,
2026-05-04 00:47:21 -03:00
"paired_event_count" : as_int( report.get( "paired_event_count" ) , 0) ,
2026-05-04 16:58:55 -03:00
"signature_expected_event_count" : signature_expected,
"signature_paired_event_count" : signature_paired,
"signature_missing_event_ids" : signature_coverage.get( "missing_event_ids" ) or [ ] ,
"signature_missing_codes" : signature_coverage.get( "missing_codes" ) or [ ] ,
"signature_unknown_pair_identity_count" : signature_unknown,
2026-05-04 14:50:47 -03:00
"paired_confidence_min" : min( pair_confidences) if pair_confidences else 0.0,
"paired_confidence_median" : median( pair_confidences) ,
2026-05-04 00:47:21 -03:00
} ,
"freshness" : {
"status" : freshness_status,
"reason" : freshness.get( "reason" , "" ) ,
"worst_event_age_p95_ms" : freshness.get( "worst_event_age_p95_ms" ) ,
2026-05-04 18:47:22 -03:00
"worst_p95_pipeline_freshness_ms" : freshness.get( "worst_p95_pipeline_freshness_ms" ) ,
"minimum_pipeline_freshness_ms" : freshness.get( "minimum_pipeline_freshness_ms" ) ,
2026-05-04 00:47:21 -03:00
"clock_uncertainty_ms" : freshness.get( "clock_uncertainty_ms" ) ,
"worst_event_age_with_uncertainty_ms" : freshness.get( "worst_event_age_with_uncertainty_ms" ) ,
"worst_freshness_drift_ms" : freshness.get( "worst_freshness_drift_ms" ) ,
"max_age_limit_ms" : freshness.get( "max_age_limit_ms" ) ,
2026-05-04 18:47:22 -03:00
"capture_timebase_status" : capture_timebase.get( "status" ) ,
"capture_timebase_valid" : capture_timebase.get( "valid" ) ,
"capture_timebase_reason" : capture_timebase.get( "reason" ) ,
2026-05-04 00:47:21 -03:00
} ,
"smoothness" : {
"video_frames" : as_int( video.get( "timestamps" ) , 0) ,
"video_p95_jitter_ms" : video_jitter,
"video_max_interval_ms" : as_float( nested( video, "interval_stats" , "max_interval_ms" ) , 0.0) ,
"video_hiccups" : video_hiccups,
"video_decoded_frames" : as_int( video.get( "decoded_frames" ) , 0) ,
"video_duplicate_frames" : duplicates,
"video_estimated_missing_frames" : missing,
"video_undecodable_frames" : undecodable,
"audio_packets" : as_int( audio_cadence.get( "timestamps" ) , 0) ,
"audio_p95_jitter_ms" : audio_jitter,
"audio_max_interval_ms" : as_float( nested( audio_cadence, "interval_stats" , "max_interval_ms" ) , 0.0) ,
"audio_hiccups" : audio_hiccups,
"audio_low_rms_windows" : low_rms,
"audio_median_rms" : as_float( nested( audio_rms, "rms_stats" , "median_rms" ) , 0.0) ,
} ,
2026-05-04 12:48:17 -03:00
"output_delay_calibration" : {
"ready" : calibration.get( "ready" ) is True,
"decision" : calibration.get( "decision" , "unknown" ) ,
"target" : calibration.get( "target" , "" ) ,
"paired_event_count" : as_int( calibration.get( "paired_event_count" ) , 0) ,
2026-05-04 14:50:47 -03:00
"min_pairs" : as_int( calibration.get( "min_pairs" ) , 0) ,
2026-05-04 12:48:17 -03:00
"measured_device_skew_ms" : as_float( calibration.get( "measured_device_skew_ms" ) , 0.0) ,
"p95_abs_skew_ms" : as_float( calibration.get( "p95_abs_skew_ms" ) , 0.0) ,
"max_abs_skew_ms" : as_float( calibration.get( "max_abs_skew_ms" ) , 0.0) ,
"drift_ms" : as_float( calibration.get( "drift_ms" ) , 0.0) ,
2026-05-04 14:35:33 -03:00
"active_audio_offset_us" : as_int( calibration.get( "active_audio_offset_us" ) , as_int( audio_delay_raw) ) ,
"active_video_offset_us" : as_int( calibration.get( "active_video_offset_us" ) , as_int( video_delay_raw) ) ,
2026-05-04 12:48:17 -03:00
"audio_offset_adjust_us" : as_int( calibration.get( "audio_offset_adjust_us" ) , 0) ,
"video_offset_adjust_us" : as_int( calibration.get( "video_offset_adjust_us" ) , 0) ,
"audio_target_offset_us" : as_int( calibration.get( "audio_target_offset_us" ) , 0) ,
"video_target_offset_us" : as_int( calibration.get( "video_target_offset_us" ) , 0) ,
2026-05-04 14:50:47 -03:00
"raw_device_delta_us" : as_int( calibration.get( "raw_device_delta_us" ) , 0) ,
"bounded_device_delta_us" : as_int( calibration.get( "bounded_device_delta_us" ) , 0) ,
"max_step_us" : as_int( calibration.get( "max_step_us" ) , 0) ,
2026-05-04 12:48:17 -03:00
"note" : calibration.get( "note" , "" ) ,
} ,
2026-05-04 00:47:21 -03:00
}
pathlib.Path( output_json) .write_text( json.dumps( artifact, indent = 2, sort_keys = True) + "\n" )
PY
}
2026-05-06 00:29:00 -03:00
annotate_mode_result( ) {
local output_json = $1
local mode_id = $2
local mode_run_index = $3
local repeat_index = $4
local repeat_count = $5
local matrix_sequence = $6
local phase = $7
python3 - <<'PY' \
" ${ output_json } " \
" ${ mode_id } " \
" ${ mode_run_index } " \
" ${ repeat_index } " \
" ${ repeat_count } " \
" ${ matrix_sequence } " \
" ${ phase } "
import json
import pathlib
import sys
(
output_json_raw,
mode_id,
mode_run_index_raw,
repeat_index_raw,
repeat_count_raw,
matrix_sequence_raw,
phase,
) = sys.argv[ 1:]
def as_int( value, default = 0) :
try:
return int( str( value) .strip( ) )
except Exception:
return default
path = pathlib.Path( output_json_raw)
result = json.loads( path.read_text( ) )
result.update(
{
"mode_id" : mode_id,
"mode_run_index" : as_int( mode_run_index_raw) ,
"repeat_index" : as_int( repeat_index_raw) ,
"repeat_count" : as_int( repeat_count_raw) ,
"matrix_sequence" : as_int( matrix_sequence_raw) ,
"probe_phase" : phase,
}
)
path.write_text( json.dumps( result, indent = 2, sort_keys = True) + "\n" )
PY
}
2026-05-04 00:47:21 -03:00
summarize_matrix( ) {
2026-05-06 00:29:00 -03:00
python3 - <<'PY' "${MATRIX_REP ORT_DIR} " " ${ MATRIX_SUMMARY_JSON } " " ${ MATRIX_SUMMARY_CSV } " " ${ MATRIX_SUMMARY_TXT } " " ${ MATRIX_DELAY_JSON } " " ${ MATRIX_DELAY_ENV } " " ${ MATRIX_STATIC_JSON } " " ${ MATRIX_STATIC_CSV } " " ${ MATRIX_STATIC_TXT } " " ${ MATRIX_STATIC_ENV } " " ${ LESAVKA_SERVER_RC_STATIC_MIN_RUNS } " " ${ LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US } " " ${ LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS } " " ${ LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS } " " ${ LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS } " " ${ LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS } "
2026-05-04 00:47:21 -03:00
import csv
import json
import pathlib
2026-05-04 14:35:33 -03:00
import shlex
2026-05-04 00:47:21 -03:00
import sys
root = pathlib.Path( sys.argv[ 1] )
summary_json = pathlib.Path( sys.argv[ 2] )
summary_csv = pathlib.Path( sys.argv[ 3] )
summary_txt = pathlib.Path( sys.argv[ 4] )
2026-05-04 14:35:33 -03:00
delay_json = pathlib.Path( sys.argv[ 5] )
delay_env = pathlib.Path( sys.argv[ 6] )
2026-05-06 00:29:00 -03:00
static_json = pathlib.Path( sys.argv[ 7] )
static_csv = pathlib.Path( sys.argv[ 8] )
static_txt = pathlib.Path( sys.argv[ 9] )
static_env = pathlib.Path( sys.argv[ 10] )
static_min_runs = int( sys.argv[ 11] )
static_max_spread_us = int( sys.argv[ 12] )
static_max_p95_skew_ms = float( sys.argv[ 13] )
static_max_median_skew_ms = float( sys.argv[ 14] )
static_require_freshness = sys.argv[ 15] .strip( ) .lower( ) not in { "" , "0" , "false" , "no" , "off" }
static_require_smoothness = sys.argv[ 16] .strip( ) .lower( ) not in { "" , "0" , "false" , "no" , "off" }
def as_int( value, default = 0) :
try:
return int( str( value) .strip( ) )
except Exception:
return default
def as_float( value, default = 0.0) :
try:
return float( str( value) .strip( ) )
except Exception:
return default
def median( values, default = None) :
cleaned = sorted( value for value in values if value is not None)
if not cleaned:
return default
mid = len( cleaned) // 2
if len( cleaned) % 2:
return cleaned[ mid]
return ( cleaned[ mid - 1] + cleaned[ mid] ) / 2.0
def median_int( values, default = None) :
value = median( values, default)
if value is None:
return None
return int( round( value) )
2026-05-04 00:47:21 -03:00
results = [ ]
for path in sorted( root.glob( "*/mode-result.json" ) ) :
try:
2026-05-04 14:35:33 -03:00
result = json.loads( path.read_text( ) )
seed_path = path.with_name( "mode-result-seed.json" )
tuned_path = path.with_name( "mode-result-tuned.json" )
if seed_path.exists( ) :
result[ "seed_result_json" ] = str( seed_path)
try:
seed = json.loads( seed_path.read_text( ) )
result[ "seed_audio_delay_us" ] = seed.get( "audio_delay_us" )
result[ "seed_video_delay_us" ] = seed.get( "video_delay_us" )
except Exception:
pass
if tuned_path.exists( ) :
result[ "tuned_result_json" ] = str( tuned_path)
2026-05-06 00:29:00 -03:00
result[ "result_json" ] = str( path)
2026-05-04 14:35:33 -03:00
results.append( result)
2026-05-04 00:47:21 -03:00
except Exception:
continue
2026-05-06 00:29:00 -03:00
results.sort(
key = lambda item: (
item.get( "mode" ) or "" ,
as_int( item.get( "mode_run_index" ) , as_int( item.get( "matrix_sequence" ) , 0) ) ,
as_int( item.get( "matrix_sequence" ) , 0) ,
)
)
2026-05-04 00:47:21 -03:00
2026-05-06 00:29:00 -03:00
mode_groups = { }
2026-05-04 14:35:33 -03:00
for result in results:
mode = result.get( "mode" )
if not mode:
continue
2026-05-06 00:29:00 -03:00
mode_groups.setdefault( mode, [ ] ) .append( result)
def static_eligibility( result) :
2026-05-04 14:35:33 -03:00
sync = result.get( "sync" ) or { }
2026-05-06 00:29:00 -03:00
freshness = result.get( "freshness" ) or { }
2026-05-04 14:35:33 -03:00
calibration = result.get( "output_delay_calibration" ) or { }
2026-05-06 00:29:00 -03:00
reasons = [ ]
required_pairs = as_int( calibration.get( "min_pairs" ) , 13) or 13
if result.get( "passed" ) is not True:
reasons.append( "probe did not pass all required gates" )
if sync.get( "passed" ) is not True:
reasons.append( f"sync did not pass ({sync.get('status', 'unknown')})" )
if as_int( sync.get( "paired_event_count" ) , 0) < required_pairs:
reasons.append( f"paired events {sync.get('paired_event_count', 0)} < {required_pairs}" )
if as_float( sync.get( "p95_abs_skew_ms" ) , 0.0) > static_max_p95_skew_ms:
reasons.append(
f"p95 skew {as_float(sync.get('p95_abs_skew_ms'), 0.0):.1f}ms > {static_max_p95_skew_ms:.1f}ms"
)
if abs( as_float( sync.get( "median_skew_ms" ) , 0.0) ) > static_max_median_skew_ms:
reasons.append(
f"median skew {as_float(sync.get('median_skew_ms'), 0.0):+.1f}ms outside +/-{static_max_median_skew_ms:.1f}ms"
)
if static_require_freshness and freshness.get( "status" ) != "pass" :
reasons.append( f"freshness did not pass ({freshness.get('status', 'unknown')})" )
if static_require_smoothness and result.get( "smoothness_warnings" ) :
reasons.append( "smoothness warnings present" )
if not isinstance( result.get( "video_delay_us" ) , int) or not isinstance( result.get( "audio_delay_us" ) , int) :
reasons.append( "tested delay values unavailable" )
return reasons
static_modes = { }
delay_recommendations = { }
video_delay_entries = [ ]
audio_delay_entries = [ ]
static_video_entries = [ ]
static_audio_entries = [ ]
for mode, mode_results in sorted( mode_groups.items( ) ) :
eligible = [ ]
rejected = [ ]
for result in mode_results:
reasons = static_eligibility( result)
if reasons:
rejected.append(
{
"mode_run_index" : result.get( "mode_run_index" ) ,
"artifact_dir" : result.get( "artifact_dir" ) ,
"reasons" : reasons,
"tested_video_delay_us" : result.get( "video_delay_us" ) ,
"tested_audio_delay_us" : result.get( "audio_delay_us" ) ,
"p95_abs_skew_ms" : ( result.get( "sync" ) or { } ) .get( "p95_abs_skew_ms" ) ,
"median_skew_ms" : ( result.get( "sync" ) or { } ) .get( "median_skew_ms" ) ,
}
)
else :
eligible.append( result)
tested_video = [ result.get( "video_delay_us" ) for result in eligible]
tested_audio = [ result.get( "audio_delay_us" ) for result in eligible]
target_video = [
( result.get( "output_delay_calibration" ) or { } ) .get( "video_target_offset_us" )
for result in eligible
if isinstance( ( result.get( "output_delay_calibration" ) or { } ) .get( "video_target_offset_us" ) , int)
]
target_audio = [
( result.get( "output_delay_calibration" ) or { } ) .get( "audio_target_offset_us" )
for result in eligible
if isinstance( ( result.get( "output_delay_calibration" ) or { } ) .get( "audio_target_offset_us" ) , int)
]
tested_video_spread = ( max( tested_video) - min( tested_video) ) if tested_video else None
tested_audio_spread = ( max( tested_audio) - min( tested_audio) ) if tested_audio else None
target_video_spread = ( max( target_video) - min( target_video) ) if target_video else None
recommended_video = median_int( tested_video)
recommended_audio = median_int( tested_audio)
status_reasons = [ ]
if len( eligible) < static_min_runs:
status_reasons.append( f"eligible runs {len(eligible)} < {static_min_runs}" )
if tested_video_spread is not None and tested_video_spread > static_max_spread_us:
status_reasons.append( f"tested video delay spread {tested_video_spread}us > {static_max_spread_us}us" )
if tested_audio_spread is not None and tested_audio_spread > static_max_spread_us:
status_reasons.append( f"tested audio delay spread {tested_audio_spread}us > {static_max_spread_us}us" )
if recommended_video is None or recommended_audio is None:
status_reasons.append( "no recommended delay could be derived" )
if len( eligible) < static_min_runs:
status = "needs_more_runs"
elif status_reasons:
status = "unstable"
else :
status = "ready"
p95_values = [ as_float( ( result.get( "sync" ) or { } ) .get( "p95_abs_skew_ms" ) , 0.0) for result in eligible]
median_skews = [ as_float( ( result.get( "sync" ) or { } ) .get( "median_skew_ms" ) , 0.0) for result in eligible]
freshness_budgets = [
as_float( ( result.get( "freshness" ) or { } ) .get( "worst_event_age_with_uncertainty_ms" ) , 0.0)
for result in eligible
]
static_entry = {
2026-05-04 14:35:33 -03:00
"status" : status,
2026-05-06 00:29:00 -03:00
"ready" : status = = "ready" ,
"reasons" : status_reasons,
"mode" : mode,
"total_runs" : len( mode_results) ,
"eligible_runs" : len( eligible) ,
"rejected_runs" : rejected,
"recommended_video_delay_us" : recommended_video,
"recommended_audio_delay_us" : recommended_audio,
"tested_video_delay_us" : tested_video,
"tested_audio_delay_us" : tested_audio,
"tested_video_delay_min_us" : min( tested_video) if tested_video else None,
"tested_video_delay_max_us" : max( tested_video) if tested_video else None,
"tested_video_delay_spread_us" : tested_video_spread,
"tested_audio_delay_spread_us" : tested_audio_spread,
"target_video_delay_us" : target_video,
"target_audio_delay_us" : target_audio,
"target_video_delay_median_us" : median_int( target_video) ,
"target_audio_delay_median_us" : median_int( target_audio) ,
"target_video_delay_spread_us" : target_video_spread,
"sync_p95_abs_skew_ms_max" : max( p95_values) if p95_values else None,
"sync_p95_abs_skew_ms_median" : median( p95_values) ,
"sync_abs_median_skew_ms_max" : max( ( abs( value) for value in median_skews) , default = None) ,
"freshness_budget_ms_max" : max( freshness_budgets) if freshness_budgets else None,
"eligible_artifact_dirs" : [ result.get( "artifact_dir" ) for result in eligible] ,
"eligible_result_json" : [ result.get( "result_json" ) for result in eligible] ,
}
static_modes[ mode] = static_entry
if status = = "ready" and isinstance( recommended_video, int) and isinstance( recommended_audio, int) :
static_video_entries.append( f"{mode}={recommended_video}" )
static_audio_entries.append( f"{mode}={recommended_audio}" )
delay_video = recommended_video
delay_audio = recommended_audio
recommendation_status = "static_ready" if status = = "ready" else status
if status != "ready" :
fallback = ( eligible or mode_results) [ -1]
fallback_calibration = fallback.get( "output_delay_calibration" ) or { }
fallback_video_target = fallback_calibration.get( "video_target_offset_us" )
fallback_audio_target = fallback_calibration.get( "audio_target_offset_us" )
if not isinstance( delay_video, int) or not isinstance( delay_audio, int) :
if (
fallback_calibration.get( "ready" ) is True
and isinstance( fallback_video_target, int)
and isinstance( fallback_audio_target, int)
) :
delay_video = fallback_video_target
delay_audio = fallback_audio_target
recommendation_status = "candidate_unconfirmed"
else :
delay_video = fallback.get( "video_delay_us" )
delay_audio = fallback.get( "audio_delay_us" )
recommendation_status = "tested_unstable"
else :
recommendation_status = f"candidate_{status}"
if isinstance( delay_video, int) and isinstance( delay_audio, int) :
video_delay_entries.append( f"{mode}={delay_video}" )
audio_delay_entries.append( f"{mode}={delay_audio}" )
delay_recommendations[ mode] = {
"status" : recommendation_status,
"static_status" : status,
"audio_delay_us" : delay_audio,
"video_delay_us" : delay_video,
"eligible_runs" : len( eligible) ,
"total_runs" : len( mode_results) ,
"tested_video_delay_spread_us" : tested_video_spread,
"sync_p95_abs_skew_ms_max" : static_entry[ "sync_p95_abs_skew_ms_max" ] ,
"sync_abs_median_skew_ms_max" : static_entry[ "sync_abs_median_skew_ms_max" ] ,
"freshness_budget_ms_max" : static_entry[ "freshness_budget_ms_max" ] ,
"reasons" : status_reasons,
2026-05-04 14:35:33 -03:00
}
2026-05-04 00:47:21 -03:00
summary = {
"schema" : "lesavka.server-rc-mode-matrix-summary.v1" ,
"artifact_dir" : str( root) ,
"passed" : bool( results) and all( result.get( "passed" ) for result in results) ,
"mode_count" : len( results) ,
2026-05-04 14:35:33 -03:00
"delay_recommendations_json" : str( delay_json) ,
"delay_recommendations_env" : str( delay_env) ,
2026-05-06 00:29:00 -03:00
"static_calibration_json" : str( static_json) ,
"static_calibration_csv" : str( static_csv) ,
"static_calibration_txt" : str( static_txt) ,
"static_calibration_env" : str( static_env) ,
"static_ready" : bool( static_modes) and all( entry.get( "ready" ) for entry in static_modes.values( ) ) ,
"static_calibration" : static_modes,
2026-05-04 14:35:33 -03:00
"delay_recommendations" : delay_recommendations,
2026-05-04 00:47:21 -03:00
"results" : results,
}
summary_json.write_text( json.dumps( summary, indent = 2, sort_keys = True) + "\n" )
2026-05-04 14:35:33 -03:00
delay_json.write_text(
json.dumps(
{
"schema" : "lesavka.server-rc-mode-delay-recommendations.v1" ,
"artifact_dir" : str( root) ,
"video_delays_us" : {
mode: entry.get( "video_delay_us" )
for mode, entry in delay_recommendations.items( )
} ,
"audio_delays_us" : {
mode: entry.get( "audio_delay_us" )
for mode, entry in delay_recommendations.items( )
} ,
"recommendations" : delay_recommendations,
} ,
indent = 2,
sort_keys = True,
)
+ "\n"
)
2026-05-06 00:29:00 -03:00
static_json.write_text(
json.dumps(
{
"schema" : "lesavka.server-rc-static-calibration.v1" ,
"artifact_dir" : str( root) ,
"ready" : summary[ "static_ready" ] ,
"criteria" : {
"min_runs" : static_min_runs,
"max_spread_us" : static_max_spread_us,
"max_p95_skew_ms" : static_max_p95_skew_ms,
"max_median_skew_ms" : static_max_median_skew_ms,
"require_freshness" : static_require_freshness,
"require_smoothness" : static_require_smoothness,
} ,
"video_delays_us" : {
mode: entry.get( "recommended_video_delay_us" )
for mode, entry in static_modes.items( )
if entry.get( "ready" )
} ,
"audio_delays_us" : {
mode: entry.get( "recommended_audio_delay_us" )
for mode, entry in static_modes.items( )
if entry.get( "ready" )
} ,
"modes" : static_modes,
} ,
indent = 2,
sort_keys = True,
)
+ "\n"
)
2026-05-04 14:35:33 -03:00
delay_env.write_text(
"LESAVKA_SERVER_RC_MODE_DELAYS_US="
+ shlex.quote( "," .join( video_delay_entries) )
+ "\n"
+ "LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US="
+ shlex.quote( "," .join( audio_delay_entries) )
+ "\n"
)
2026-05-06 00:29:00 -03:00
static_env.write_text(
"LESAVKA_SERVER_RC_MODE_DELAYS_US="
+ shlex.quote( "," .join( static_video_entries) )
+ "\n"
+ "LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US="
+ shlex.quote( "," .join( static_audio_entries) )
+ "\n"
)
2026-05-04 00:47:21 -03:00
fieldnames = [
"mode" ,
2026-05-06 00:29:00 -03:00
"mode_run_index" ,
"repeat_index" ,
"matrix_sequence" ,
2026-05-04 00:47:21 -03:00
"passed" ,
2026-05-04 14:35:33 -03:00
"seed_video_delay_us" ,
"seed_audio_delay_us" ,
2026-05-04 00:47:21 -03:00
"video_delay_us" ,
2026-05-04 12:48:17 -03:00
"audio_delay_us" ,
2026-05-04 00:47:21 -03:00
"sync_status" ,
"p95_abs_skew_ms" ,
"median_skew_ms" ,
"sync_drift_ms" ,
"freshness_status" ,
"freshness_budget_ms" ,
"freshness_drift_ms" ,
2026-05-04 18:47:22 -03:00
"capture_timebase_status" ,
2026-05-04 00:47:21 -03:00
"video_hiccups" ,
"video_missing" ,
"video_undecodable" ,
"audio_hiccups" ,
"audio_low_rms" ,
2026-05-04 12:48:17 -03:00
"calibration_ready" ,
"calibration_decision" ,
"calibration_target" ,
"calibration_measured_skew_ms" ,
"calibration_video_target_offset_us" ,
"calibration_audio_target_offset_us" ,
2026-05-04 00:47:21 -03:00
"artifact_dir" ,
2026-05-06 00:29:00 -03:00
"result_json" ,
2026-05-04 00:47:21 -03:00
]
with summary_csv.open( "w" , newline = "" , encoding = "utf-8" ) as handle:
writer = csv.DictWriter( handle, fieldnames = fieldnames)
writer.writeheader( )
for result in results:
writer.writerow( {
"mode" : result.get( "mode" ) ,
2026-05-06 00:29:00 -03:00
"mode_run_index" : result.get( "mode_run_index" ) ,
"repeat_index" : result.get( "repeat_index" ) ,
"matrix_sequence" : result.get( "matrix_sequence" ) ,
2026-05-04 00:47:21 -03:00
"passed" : result.get( "passed" ) ,
2026-05-04 14:35:33 -03:00
"seed_video_delay_us" : result.get( "seed_video_delay_us" ) ,
"seed_audio_delay_us" : result.get( "seed_audio_delay_us" ) ,
2026-05-04 00:47:21 -03:00
"video_delay_us" : result.get( "video_delay_us" ) ,
2026-05-04 12:48:17 -03:00
"audio_delay_us" : result.get( "audio_delay_us" ) ,
2026-05-04 00:47:21 -03:00
"sync_status" : ( result.get( "sync" ) or { } ) .get( "status" ) ,
"p95_abs_skew_ms" : ( result.get( "sync" ) or { } ) .get( "p95_abs_skew_ms" ) ,
"median_skew_ms" : ( result.get( "sync" ) or { } ) .get( "median_skew_ms" ) ,
"sync_drift_ms" : ( result.get( "sync" ) or { } ) .get( "drift_ms" ) ,
"freshness_status" : ( result.get( "freshness" ) or { } ) .get( "status" ) ,
"freshness_budget_ms" : ( result.get( "freshness" ) or { } ) .get( "worst_event_age_with_uncertainty_ms" ) ,
"freshness_drift_ms" : ( result.get( "freshness" ) or { } ) .get( "worst_freshness_drift_ms" ) ,
2026-05-04 18:47:22 -03:00
"capture_timebase_status" : ( result.get( "freshness" ) or { } ) .get( "capture_timebase_status" ) ,
2026-05-04 00:47:21 -03:00
"video_hiccups" : ( result.get( "smoothness" ) or { } ) .get( "video_hiccups" ) ,
"video_missing" : ( result.get( "smoothness" ) or { } ) .get( "video_estimated_missing_frames" ) ,
"video_undecodable" : ( result.get( "smoothness" ) or { } ) .get( "video_undecodable_frames" ) ,
"audio_hiccups" : ( result.get( "smoothness" ) or { } ) .get( "audio_hiccups" ) ,
"audio_low_rms" : ( result.get( "smoothness" ) or { } ) .get( "audio_low_rms_windows" ) ,
2026-05-04 12:48:17 -03:00
"calibration_ready" : ( result.get( "output_delay_calibration" ) or { } ) .get( "ready" ) ,
"calibration_decision" : ( result.get( "output_delay_calibration" ) or { } ) .get( "decision" ) ,
"calibration_target" : ( result.get( "output_delay_calibration" ) or { } ) .get( "target" ) ,
"calibration_measured_skew_ms" : ( result.get( "output_delay_calibration" ) or { } ) .get( "measured_device_skew_ms" ) ,
"calibration_video_target_offset_us" : ( result.get( "output_delay_calibration" ) or { } ) .get( "video_target_offset_us" ) ,
"calibration_audio_target_offset_us" : ( result.get( "output_delay_calibration" ) or { } ) .get( "audio_target_offset_us" ) ,
2026-05-04 00:47:21 -03:00
"artifact_dir" : result.get( "artifact_dir" ) ,
2026-05-06 00:29:00 -03:00
"result_json" : result.get( "result_json" ) ,
2026-05-04 00:47:21 -03:00
} )
2026-05-06 00:29:00 -03:00
static_fieldnames = [
"mode" ,
"status" ,
"ready" ,
"eligible_runs" ,
"total_runs" ,
"recommended_video_delay_us" ,
"recommended_audio_delay_us" ,
"tested_video_delay_min_us" ,
"tested_video_delay_max_us" ,
"tested_video_delay_spread_us" ,
"target_video_delay_median_us" ,
"target_video_delay_spread_us" ,
"sync_p95_abs_skew_ms_max" ,
"sync_p95_abs_skew_ms_median" ,
"sync_abs_median_skew_ms_max" ,
"freshness_budget_ms_max" ,
"reasons" ,
]
with static_csv.open( "w" , newline = "" , encoding = "utf-8" ) as handle:
writer = csv.DictWriter( handle, fieldnames = static_fieldnames)
writer.writeheader( )
for mode, entry in sorted( static_modes.items( ) ) :
writer.writerow(
{
"mode" : mode,
"status" : entry.get( "status" ) ,
"ready" : entry.get( "ready" ) ,
"eligible_runs" : entry.get( "eligible_runs" ) ,
"total_runs" : entry.get( "total_runs" ) ,
"recommended_video_delay_us" : entry.get( "recommended_video_delay_us" ) ,
"recommended_audio_delay_us" : entry.get( "recommended_audio_delay_us" ) ,
"tested_video_delay_min_us" : entry.get( "tested_video_delay_min_us" ) ,
"tested_video_delay_max_us" : entry.get( "tested_video_delay_max_us" ) ,
"tested_video_delay_spread_us" : entry.get( "tested_video_delay_spread_us" ) ,
"target_video_delay_median_us" : entry.get( "target_video_delay_median_us" ) ,
"target_video_delay_spread_us" : entry.get( "target_video_delay_spread_us" ) ,
"sync_p95_abs_skew_ms_max" : entry.get( "sync_p95_abs_skew_ms_max" ) ,
"sync_p95_abs_skew_ms_median" : entry.get( "sync_p95_abs_skew_ms_median" ) ,
"sync_abs_median_skew_ms_max" : entry.get( "sync_abs_median_skew_ms_max" ) ,
"freshness_budget_ms_max" : entry.get( "freshness_budget_ms_max" ) ,
"reasons" : "; " .join( entry.get( "reasons" ) or [ ] ) ,
}
)
2026-05-04 00:47:21 -03:00
lines = [
f"Server-to-RC mode matrix for {root}" ,
2026-05-06 00:29:00 -03:00
f"- runs: {len(results)}" ,
f"- unique modes: {len(mode_groups)}" ,
2026-05-04 00:47:21 -03:00
f"- verdict: {'pass' if summary['passed'] else 'fail'}" ,
2026-05-06 00:29:00 -03:00
f"- static calibration: {'ready' if summary['static_ready'] else 'needs-more-data'}" ,
2026-05-04 00:47:21 -03:00
]
for result in results:
sync = result.get( "sync" ) or { }
freshness = result.get( "freshness" ) or { }
smooth = result.get( "smoothness" ) or { }
2026-05-04 12:48:17 -03:00
calibration = result.get( "output_delay_calibration" ) or { }
2026-05-04 00:47:21 -03:00
marker = "PASS" if result.get( "passed" ) else "FAIL"
lines.append(
2026-05-06 00:29:00 -03:00
f"- {marker} {result.get('mode')} run={result.get('mode_run_index', '?')}: "
2026-05-04 12:48:17 -03:00
f"delays video={result.get('video_delay_us', 0)}us audio={result.get('audio_delay_us', 0)}us; "
2026-05-04 00:47:21 -03:00
f"sync {sync.get('status')} p95={sync.get('p95_abs_skew_ms', 0.0):.1f}ms median={sync.get('median_skew_ms', 0.0):+.1f}ms; "
f"freshness {freshness.get('status')} budget={freshness.get('worst_event_age_with_uncertainty_ms') or 0.0:.1f}ms; "
f"smooth video hiccups={smooth.get('video_hiccups', 0)} missing={smooth.get('video_estimated_missing_frames', 0)} undecodable={smooth.get('video_undecodable_frames', 0)} audio hiccups={smooth.get('audio_hiccups', 0)}"
)
2026-05-04 12:48:17 -03:00
lines.append(
" calibration: "
f"{calibration.get('decision', 'unknown')} ready={calibration.get('ready', False)} "
f"target={calibration.get('target', '')} measured={calibration.get('measured_device_skew_ms', 0.0):+.1f}ms "
f"video_target={calibration.get('video_target_offset_us', 0)}us "
f"audio_target={calibration.get('audio_target_offset_us', 0)}us"
)
2026-05-04 14:50:47 -03:00
lines.append(
" sync evidence: "
f"video_onsets={sync.get('video_event_count', 0)} audio_onsets={sync.get('audio_event_count', 0)} "
f"pairs={sync.get('paired_event_count', 0)} "
2026-05-04 16:58:55 -03:00
f"coded_pairs={sync.get('signature_paired_event_count', 0)}/{sync.get('signature_expected_event_count', 0)} "
f"missing_codes={sync.get('signature_missing_codes') or []} "
f"unknown_identity={sync.get('signature_unknown_pair_identity_count', 0)} "
2026-05-04 14:50:47 -03:00
f"pair_conf_median={sync.get('paired_confidence_median', 0.0):.3f} "
f"raw_pair_disagreement={sync.get('activity_pair_disagreement_ms', 0.0):+.1f}ms"
)
2026-05-05 19:33:56 -03:00
visibility = result.get( "event_visibility" ) or { }
visibility_summary = visibility.get( "summary" ) or { }
if visibility_summary:
lines.append(
" coded visibility: "
f"paired={visibility_summary.get('paired', 0)} "
f"video_only={visibility_summary.get('video_only', 0)} "
f"audio_only={visibility_summary.get('audio_only', 0)} "
f"unpaired={visibility_summary.get('unpaired', 0)} "
f"missing={visibility_summary.get('missing', 0)}"
)
2026-05-04 18:47:22 -03:00
if freshness.get( "capture_timebase_status" ) :
lines.append(
" capture timing: "
f"{freshness.get('capture_timebase_status')} "
f"valid={freshness.get('capture_timebase_valid')} "
f"reason={freshness.get('capture_timebase_reason', '')}"
)
2026-05-05 20:27:37 -03:00
signal_readiness = result.get( "signal_readiness" ) or { }
signal_attempts = signal_readiness.get( "attempts" ) or [ ]
if signal_readiness:
lines.append(
" signal readiness: "
f"passed={signal_readiness.get('passed', False)} "
f"attempts={len(signal_attempts)} "
f"reason={signal_readiness.get('reason', '')}"
)
for attempt in signal_attempts:
counts = attempt.get( "counts" ) or { }
layers = attempt.get( "layers" ) or { }
lines.append(
" signal attempt "
f"{attempt.get('attempt', '?')}: passed={attempt.get('passed', False)} "
f"server_events={counts.get('server_events', 0)} "
f"video={counts.get('video_events', 0)} audio={counts.get('audio_events', 0)} "
f"paired={counts.get('paired_events', counts.get('coded_pairs', 0))} "
f"audio_ready={layers.get('tethys_audio_ready', False)} "
f"video_ready={layers.get('tethys_video_ready', False)} "
f"reason={attempt.get('reason', '')}"
)
2026-05-04 14:35:33 -03:00
if "seed_video_delay_us" in result or "seed_audio_delay_us" in result:
lines.append(
" tuning: "
f"seed video={result.get('seed_video_delay_us', 0)}us audio={result.get('seed_audio_delay_us', 0)}us -> "
f"tested video={result.get('video_delay_us', 0)}us audio={result.get('audio_delay_us', 0)}us"
)
2026-05-04 00:47:21 -03:00
for reason in result.get( "failure_reasons" ) or [ ] :
lines.append( f" reason: {reason}" )
2026-05-05 19:33:56 -03:00
for warning in result.get( "smoothness_warnings" ) or [ ] :
lines.append( f" smoothness warning: {warning}" )
2026-05-04 14:35:33 -03:00
if video_delay_entries or audio_delay_entries:
lines.append( f"- recommended video delays: {','.join(video_delay_entries)}" )
lines.append( f"- recommended audio delays: {','.join(audio_delay_entries)}" )
2026-05-06 00:29:00 -03:00
static_lines = [
f"Server-to-RC static calibration for {root}" ,
f"- verdict: {'ready' if summary['static_ready'] else 'needs-more-data'}" ,
(
"- criteria: "
f"min_runs={static_min_runs} "
f"max_spread_us={static_max_spread_us} "
f"max_p95_skew_ms={static_max_p95_skew_ms:.1f} "
f"max_median_skew_ms={static_max_median_skew_ms:.1f} "
f"require_freshness={static_require_freshness} "
f"require_smoothness={static_require_smoothness}"
) ,
]
for mode, entry in sorted( static_modes.items( ) ) :
marker = "READY" if entry.get( "ready" ) else "HOLD"
reasons = "; " .join( entry.get( "reasons" ) or [ ] )
if not reasons and entry.get( "ready" ) :
reasons = "all stability criteria met"
static_lines.append(
f"- {marker} {mode}: "
f"runs={entry.get('eligible_runs')}/{entry.get('total_runs')} "
f"video={entry.get('recommended_video_delay_us')}us "
f"audio={entry.get('recommended_audio_delay_us')}us "
f"spread={entry.get('tested_video_delay_spread_us')}us "
f"p95_max={entry.get('sync_p95_abs_skew_ms_max') or 0.0:.1f}ms "
f"median_abs_max={entry.get('sync_abs_median_skew_ms_max') or 0.0:.1f}ms "
f"freshness_budget_max={entry.get('freshness_budget_ms_max') or 0.0:.1f}ms"
)
static_lines.append( f" reason: {reasons}" )
if static_video_entries or static_audio_entries:
static_lines.append( f"- static video delays: {','.join(static_video_entries)}" )
static_lines.append( f"- static audio delays: {','.join(static_audio_entries)}" )
lines.extend( [ "" , *static_lines] )
2026-05-04 00:47:21 -03:00
summary_txt.write_text( "\n" .join( lines) + "\n" )
2026-05-06 00:29:00 -03:00
static_txt.write_text( "\n" .join( static_lines) + "\n" )
2026-05-04 00:47:21 -03:00
print( "\n" .join( lines) )
PY
}
2026-05-04 12:48:17 -03:00
if [ [ " ${ LESAVKA_SERVER_RC_MODES } " = = "auto" ] ] ; then
echo "==> discovering local webcam-backed Lesavka modes"
LESAVKA_SERVER_RC_MODES = " $( discover_local_webcam_modes) "
LESAVKA_SERVER_RC_MODE_SOURCE = local-v4l2
fi
2026-05-04 00:47:21 -03:00
echo "==> server-to-RC mode matrix"
echo " ↪ modes= ${ LESAVKA_SERVER_RC_MODES } "
2026-05-04 12:48:17 -03:00
echo " ↪ mode_source= ${ LESAVKA_SERVER_RC_MODE_SOURCE } "
2026-05-06 00:29:00 -03:00
echo " ↪ repeat_count= ${ LESAVKA_SERVER_RC_REPEAT_COUNT } verbose_probes= ${ LESAVKA_SERVER_RC_VERBOSE_PROBES } "
2026-05-04 12:48:17 -03:00
echo " ↪ video_delays= ${ LESAVKA_SERVER_RC_MODE_DELAYS_US } "
echo " ↪ audio_delays= ${ LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US } "
2026-05-04 14:05:55 -03:00
echo " ↪ capture_stack= ${ REMOTE_CAPTURE_STACK } audio_source= ${ REMOTE_AUDIO_SOURCE } pulse_tool= ${ REMOTE_PULSE_CAPTURE_TOOL } video_mode= ${ REMOTE_PULSE_VIDEO_MODE } "
2026-05-04 14:50:47 -03:00
echo " ↪ tune_delays= ${ LESAVKA_SERVER_RC_TUNE_DELAYS } confirm= ${ LESAVKA_SERVER_RC_TUNE_CONFIRM } min_pairs= ${ LESAVKA_SERVER_RC_TUNE_MIN_PAIRS } max_abs_skew_ms= ${ LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS } max_step_us= ${ LESAVKA_SERVER_RC_TUNE_MAX_STEP_US } min_change_us= ${ LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US } "
2026-05-06 00:29:00 -03:00
echo " ↪ static_min_runs= ${ LESAVKA_SERVER_RC_STATIC_MIN_RUNS } static_max_spread_us= ${ LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US } static_max_p95_skew_ms= ${ LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS } static_max_median_skew_ms= ${ LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS } "
2026-05-04 20:22:57 -03:00
echo " ↪ freshness_limit_ms= ${ LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS } min_pairs= ${ LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS } "
2026-05-05 19:33:56 -03:00
echo " ↪ coded_pairs_min= ${ LESAVKA_SERVER_RC_MIN_CODED_PAIRS } require_all_coded= ${ LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS } smoothness_gate= ${ LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS } "
2026-05-05 21:01:05 -03:00
echo " ↪ signal_ready= ${ LESAVKA_SERVER_RC_SIGNAL_READY } mode= ${ LESAVKA_SERVER_RC_SIGNAL_READY_MODE } attempts= ${ LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS } min_pairs= ${ LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS } duration= ${ LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS } s warmup= ${ LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS } s retry_delay= ${ LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS } s "
echo " ↪ signal_conditioning= ${ LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS } s warmup= ${ LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS } s gap= ${ LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS } s analysis_window= ${ LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW } "
2026-05-04 12:48:17 -03:00
echo " ↪ reconfigure= ${ LESAVKA_SERVER_RC_RECONFIGURE } strategy= ${ LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY } allow_gadget_reset= ${ LESAVKA_SERVER_RC_ALLOW_GADGET_RESET } "
echo " ↪ tethys_ready= ${ LESAVKA_SERVER_RC_WAIT_TETHYS_READY } settle= ${ LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS } s timeout= ${ LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS } s preroll_discard= ${ LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS } s "
2026-05-04 20:32:21 -03:00
echo " ↪ start_delay= ${ LESAVKA_SERVER_RC_START_DELAY_SECONDS } s "
2026-05-04 00:47:21 -03:00
echo " ↪ artifact_dir= ${ MATRIX_REPORT_DIR } "
2026-05-06 00:29:00 -03:00
echo " ↪ matrix_run_log= ${ MATRIX_RUN_LOG } "
2026-05-04 00:47:21 -03:00
2026-05-04 12:48:17 -03:00
prime_remote_sudo
2026-05-04 20:32:21 -03:00
sleep_start_delay
2026-05-04 12:48:17 -03:00
prebuild_probe_tools
2026-05-06 00:29:00 -03:00
if ! [ [ " ${ LESAVKA_SERVER_RC_REPEAT_COUNT } " = ~ ^[ 0-9] +$ ] ] || ( ( LESAVKA_SERVER_RC_REPEAT_COUNT < 1 ) ) ; then
printf 'LESAVKA_SERVER_RC_REPEAT_COUNT must be a positive integer; got %s\n' " ${ LESAVKA_SERVER_RC_REPEAT_COUNT } " >& 2
exit 64
fi
declare -A mode_run_counts = ( )
matrix_sequence = 0
2026-05-04 00:47:21 -03:00
IFS = ',' read -r -a modes <<< " ${ LESAVKA_SERVER_RC_MODES } "
for mode in " ${ modes [@] } " ; do
mode = " ${ mode //[[ : space : ]]/ } "
[ [ -n " ${ mode } " ] ] || continue
read -r width height fps < <( parse_mode " ${ mode } " )
2026-05-05 21:01:05 -03:00
2026-05-06 00:29:00 -03:00
for repeat_index in $( seq 1 " ${ LESAVKA_SERVER_RC_REPEAT_COUNT } " ) ; do
mode_run_counts[ " ${ mode } " ] = $(( ${ mode_run_counts [ " ${ mode } " ] :- 0 } + 1 ))
mode_run_index = ${ mode_run_counts [ " ${ mode } " ] }
matrix_sequence = $(( matrix_sequence + 1 ))
video_delay_us = " $( lookup_video_delay_us " ${ mode } " ) "
audio_delay_us = " $( lookup_audio_delay_us " ${ mode } " ) "
id_base = " $( mode_id " ${ mode } " ) "
run_label = " $( printf '%02d' " ${ mode_run_index } " ) "
id = " ${ id_base } __run ${ run_label } "
mode_dir = " ${ MATRIX_REPORT_DIR } / ${ id } "
mode_log = " ${ mode_dir } /mode-run.log "
mode_result = " ${ mode_dir } /mode-result.json "
seed_result = " ${ mode_dir } /mode-result-seed.json "
readiness_dir = " ${ mode_dir } /signal-readiness "
readiness_log = " ${ readiness_dir } /signal-readiness-run.log "
readiness_attempts_json = " ${ readiness_dir } /signal-readiness-attempts.json "
tuned_log = " ${ mode_dir } /mode-tuned-run.log "
tuned_result = " ${ mode_dir } /mode-result-tuned.json "
tune_env = " ${ mode_dir } /mode-tune-candidate.env "
mkdir -p " ${ mode_dir } "
echo " ==> mode ${ mode } run ${ mode_run_index } repeat ${ repeat_index } / ${ LESAVKA_SERVER_RC_REPEAT_COUNT } : video_delay_us= ${ video_delay_us } audio_delay_us= ${ audio_delay_us } "
2026-05-06 03:59:20 -03:00
reconfigure_server_mode " ${ mode } " " ${ width } " " ${ height } " " ${ fps } " " ${ audio_delay_us } " " ${ video_delay_us } "
2026-05-06 00:29:00 -03:00
wait_tethys_media_ready " ${ mode } " " ${ width } " " ${ height } " " ${ fps } "
if [ [ " ${ LESAVKA_SERVER_RC_SIGNAL_READY } " != "0" && " ${ LESAVKA_SERVER_RC_SIGNAL_READY_MODE } " != "separate" ] ] ; then
echo " ==> mode ${ mode } run ${ mode_run_index } : using same-capture signal conditioning before measured probe "
fi
if [ [ " ${ LESAVKA_SERVER_RC_SIGNAL_READY } " != "0" && " ${ LESAVKA_SERVER_RC_SIGNAL_READY_MODE } " = = "separate" ] ] ; then
mkdir -p " ${ readiness_dir } "
echo " ==> mode ${ mode } run ${ mode_run_index } : proving Tethys signal readiness before measured probe "
readiness_pass = 1
readiness_status = 0
readiness_artifact_dir = ""
readiness_reason = "no readiness attempts ran"
readiness_attempt_jsons = ( )
for readiness_attempt in $( seq 1 " ${ LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS } " ) ; do
attempt_dir = " ${ readiness_dir } /attempt- ${ readiness_attempt } "
readiness_log = " ${ attempt_dir } /signal-readiness-run.log "
readiness_attempt_json = " ${ attempt_dir } /signal-readiness-attempt.json "
mkdir -p " ${ attempt_dir } "
echo " ↪ readiness attempt ${ readiness_attempt } / ${ LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS } : requiring ${ LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS } paired coded events "
run_mode_probe \
" ${ width } " \
" ${ height } " \
" ${ fps } " \
" ${ audio_delay_us } " \
" ${ video_delay_us } " \
" ${ attempt_dir } " \
2026-05-05 20:27:37 -03:00
" ${ readiness_log } " \
2026-05-06 00:29:00 -03:00
" ${ LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS } " \
" ${ LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS } " \
2026-05-05 20:27:37 -03:00
" ${ LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS } " \
2026-05-06 00:29:00 -03:00
0
readiness_status = ${ RUN_MODE_PROBE_STATUS }
readiness_artifact_dir = " $( artifact_dir_from_log " ${ readiness_log } " " ${ attempt_dir } " ) "
set +e
readiness_reason = " $(
write_signal_readiness_attempt_result \
" ${ readiness_attempt } " \
" ${ readiness_artifact_dir } " \
" ${ readiness_status } " \
" ${ readiness_log } " \
" ${ LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS } " \
" ${ readiness_attempt_json } " 2>& 1
) "
readiness_pass = $?
set -e
readiness_attempt_jsons += ( " ${ readiness_attempt_json } " )
if [ [ " ${ readiness_pass } " -eq 0 ] ] ; then
echo " ↪ readiness attempt ${ readiness_attempt } passed: ${ readiness_reason } "
break
fi
[ [ -n " ${ readiness_reason } " ] ] || readiness_reason = "signal readiness failed"
echo " ↪ readiness attempt ${ readiness_attempt } failed: ${ readiness_reason } "
if [ [ " ${ readiness_attempt } " -lt " ${ LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS } " ] ] ; then
echo " ↪ waiting ${ LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS } s before retrying signal readiness "
sleep " ${ LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS } "
fi
done
write_signal_readiness_attempts_summary " ${ readiness_attempts_json } " " ${ readiness_attempt_jsons [@] } "
if [ [ " ${ readiness_pass } " -ne 0 ] ] ; then
[ [ -n " ${ readiness_reason } " ] ] || readiness_reason = "signal readiness failed"
echo " ↪ signal readiness failed: ${ readiness_reason } "
write_signal_readiness_failure \
" ${ mode } " \
" ${ width } " \
" ${ height } " \
" ${ fps } " \
" ${ video_delay_us } " \
" ${ audio_delay_us } " \
" ${ readiness_status } " \
" ${ readiness_log } " \
" ${ readiness_artifact_dir } " \
" ${ readiness_attempts_json } " \
" ${ readiness_reason } " \
" ${ mode_result } "
annotate_mode_result " ${ mode_result } " " ${ id_base } " " ${ mode_run_index } " " ${ repeat_index } " " ${ LESAVKA_SERVER_RC_REPEAT_COUNT } " " ${ matrix_sequence } " "readiness-failure"
cp " ${ mode_result } " " ${ seed_result } "
if [ [ " ${ LESAVKA_SERVER_RC_CONTINUE_ON_FAIL } " = = "0" ] ] ; then
break 2
fi
continue
2026-05-05 19:33:56 -03:00
fi
2026-05-06 00:29:00 -03:00
echo " ↪ signal readiness passed: artifact_dir= ${ readiness_artifact_dir } "
2026-05-05 19:33:56 -03:00
fi
2026-05-06 00:29:00 -03:00
echo " ==> mode ${ mode } run ${ mode_run_index } : running seed probe "
run_mode_probe " ${ width } " " ${ height } " " ${ fps } " " ${ audio_delay_us } " " ${ video_delay_us } " " ${ mode_dir } " " ${ mode_log } "
run_status = ${ RUN_MODE_PROBE_STATUS }
artifact_dir = " $( artifact_dir_from_log " ${ mode_log } " " ${ mode_dir } " ) "
write_mode_result " ${ mode } " " ${ width } " " ${ height } " " ${ fps } " " ${ video_delay_us } " " ${ audio_delay_us } " " ${ run_status } " " ${ mode_log } " " ${ artifact_dir } " " ${ mode_result } "
annotate_mode_result " ${ mode_result } " " ${ id_base } " " ${ mode_run_index } " " ${ repeat_index } " " ${ LESAVKA_SERVER_RC_REPEAT_COUNT } " " ${ matrix_sequence } " "seed"
cp " ${ mode_result } " " ${ seed_result } "
if [ [ " ${ LESAVKA_SERVER_RC_TUNE_DELAYS } " != "0" && " ${ LESAVKA_SERVER_RC_TUNE_CONFIRM } " != "0" ] ] ; then
write_tune_candidate_env " ${ seed_result } " " ${ tune_env } "
# shellcheck disable=SC1090
source " ${ tune_env } "
if [ [ " ${ tune_ready :- false } " = = "true" ] ] ; then
echo " ==> mode ${ mode } run ${ mode_run_index } : confirming tuned delays video_delay_us= ${ tune_video_delay_us } audio_delay_us= ${ tune_audio_delay_us } "
echo " ↪ tune delta: video= ${ tune_video_delta_us } us audio= ${ tune_audio_delta_us } us pairs= ${ tune_paired_event_count } drift= ${ tune_drift_ms } ms "
run_mode_probe " ${ width } " " ${ height } " " ${ fps } " " ${ tune_audio_delay_us } " " ${ tune_video_delay_us } " " ${ mode_dir } " " ${ tuned_log } "
tuned_status = ${ RUN_MODE_PROBE_STATUS }
tuned_artifact_dir = " $( artifact_dir_from_log " ${ tuned_log } " " ${ mode_dir } " ) "
write_mode_result " ${ mode } " " ${ width } " " ${ height } " " ${ fps } " " ${ tune_video_delay_us } " " ${ tune_audio_delay_us } " " ${ tuned_status } " " ${ tuned_log } " " ${ tuned_artifact_dir } " " ${ tuned_result } "
annotate_mode_result " ${ tuned_result } " " ${ id_base } " " ${ mode_run_index } " " ${ repeat_index } " " ${ LESAVKA_SERVER_RC_REPEAT_COUNT } " " ${ matrix_sequence } " "tuned"
cp " ${ tuned_result } " " ${ mode_result } "
run_status = ${ tuned_status }
else
echo " ↪ tune skipped: ${ tune_reason :- not ready } "
fi
2026-05-04 14:35:33 -03:00
fi
2026-05-04 00:47:21 -03:00
2026-05-06 00:29:00 -03:00
if [ [ " ${ run_status } " -ne 0 && " ${ LESAVKA_SERVER_RC_CONTINUE_ON_FAIL } " = = "0" ] ] ; then
break 2
fi
done
2026-05-04 00:47:21 -03:00
done
summarize_matrix
echo "==> done"
echo " artifact_dir: ${ MATRIX_REPORT_DIR } "
echo " mode_matrix_summary_json: ${ MATRIX_SUMMARY_JSON } "
echo " mode_matrix_summary_csv: ${ MATRIX_SUMMARY_CSV } "
echo " mode_matrix_summary_txt: ${ MATRIX_SUMMARY_TXT } "
2026-05-04 14:35:33 -03:00
echo " mode_delay_recommendations_json: ${ MATRIX_DELAY_JSON } "
echo " mode_delay_recommendations_env: ${ MATRIX_DELAY_ENV } "
2026-05-06 00:29:00 -03:00
echo " mode_static_calibration_json: ${ MATRIX_STATIC_JSON } "
echo " mode_static_calibration_csv: ${ MATRIX_STATIC_CSV } "
echo " mode_static_calibration_txt: ${ MATRIX_STATIC_TXT } "
echo " mode_static_calibration_env: ${ MATRIX_STATIC_ENV } "
echo " mode_matrix_run_log: ${ MATRIX_RUN_LOG } "
2026-05-04 00:47:21 -03:00
if python3 - <<'PY' " ${ MATRIX_SUMMARY_JSON } "
import json
import pathlib
import sys
summary = json.loads( pathlib.Path( sys.argv[ 1] ) .read_text( ) )
raise SystemExit( 0 if summary.get( "passed" ) else 1)
PY
then
exit 0
fi
exit 95