559 lines
29 KiB
Rust
559 lines
29 KiB
Rust
//! Contract tests for the manual upstream A/V sync harness.
|
|
//!
|
|
//! Scope: statically guard the workstation-side tunnel/bootstrap behavior.
|
|
//! Targets: `scripts/manual/run_upstream_av_sync.sh`.
|
|
//! Why: the manual probe should reach Theia through SSH even when the gRPC
|
|
//! port is not exposed on the public SSH endpoint.
|
|
|
|
const SYNC_SCRIPT: &str = include_str!("../../scripts/manual/run_upstream_av_sync.sh");
|
|
const SERVER_RC_MODE_MATRIX_SCRIPT: &str =
|
|
include_str!("../../scripts/manual/run_server_to_rc_mode_matrix.sh");
|
|
const BROWSER_SYNC_SCRIPT: &str =
|
|
include_str!("../../scripts/manual/run_upstream_browser_av_sync.sh");
|
|
const MIRRORED_SYNC_SCRIPT: &str =
|
|
include_str!("../../scripts/manual/run_upstream_mirrored_av_sync.sh");
|
|
const LOCAL_STIMULUS: &str = include_str!("../../scripts/manual/local_av_stimulus.py");
|
|
const SYNC_PROBE_RUNNER: &str = include_str!("../../client/src/sync_probe/runner.rs");
|
|
|
|
#[test]
|
|
fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
|
|
for expected in [
|
|
"cleanup_server_tunnel",
|
|
"pick_local_server_tunnel_port",
|
|
"wait_for_server_tunnel",
|
|
"start_server_tunnel",
|
|
"ExitOnForwardFailure=yes",
|
|
"127.0.0.1:${local_port}:127.0.0.1:${remote_port}",
|
|
"LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https}",
|
|
"LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}",
|
|
"RESOLVED_LESAVKA_SERVER_ADDR=\"${LESAVKA_SERVER_SCHEME}://127.0.0.1:${SERVER_TUNNEL_LOCAL_PORT}\"",
|
|
"LESAVKA_TLS_DOMAIN=\"${LESAVKA_TLS_DOMAIN}\"",
|
|
"tunneled to ${LESAVKA_SERVER_HOST}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT}",
|
|
"CAPTURE_READY_MARKER=\"__LESAVKA_CAPTURE_READY__\"",
|
|
"LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}",
|
|
"LOCAL_REPORT_DIR=\"${LOCAL_OUTPUT_DIR%/}/lesavka-output-delay-probe-${STAMP}\"",
|
|
"LOCAL_ANALYSIS_JSON=\"${LOCAL_REPORT_DIR}/report.json\"",
|
|
"LOCAL_EVENTS_CSV=\"${LOCAL_REPORT_DIR}/events.csv\"",
|
|
"LOCAL_SERVER_PROBE_REPLY=\"${LOCAL_REPORT_DIR}/server-output-probe-reply.txt\"",
|
|
"LOCAL_SERVER_TIMELINE_JSON=\"${LOCAL_REPORT_DIR}/server-output-timeline.json\"",
|
|
"LOCAL_OUTPUT_DELAY_CORRELATION_JSON=\"${LOCAL_REPORT_DIR}/output-delay-correlation.json\"",
|
|
"LOCAL_OUTPUT_DELAY_CORRELATION_CSV=\"${LOCAL_REPORT_DIR}/output-delay-correlation.csv\"",
|
|
"LOCAL_OUTPUT_DELAY_CORRELATION_TXT=\"${LOCAL_REPORT_DIR}/output-delay-correlation.txt\"",
|
|
"LOCAL_CLOCK_ALIGNMENT_JSON=\"${LOCAL_REPORT_DIR}/clock-alignment.json\"",
|
|
"LOCAL_OUTPUT_DELAY_JSON=\"${LOCAL_REPORT_DIR}/output-delay-calibration.json\"",
|
|
"LOCAL_OUTPUT_DELAY_ENV=\"${LOCAL_REPORT_DIR}/output-delay-calibration.env\"",
|
|
"LESAVKA_OUTPUT_DELAY_CALIBRATION=${LESAVKA_OUTPUT_DELAY_CALIBRATION:-1}",
|
|
"LESAVKA_OUTPUT_DELAY_APPLY=${LESAVKA_OUTPUT_DELAY_APPLY:-0}",
|
|
"LESAVKA_OUTPUT_DELAY_APPLY_MODE=${LESAVKA_OUTPUT_DELAY_APPLY_MODE:-absolute}",
|
|
"LESAVKA_OUTPUT_DELAY_CONFIRM=${LESAVKA_OUTPUT_DELAY_CONFIRM:-1}",
|
|
"LESAVKA_OUTPUT_DELAY_SAVE=${LESAVKA_OUTPUT_DELAY_SAVE:-0}",
|
|
"LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=${LESAVKA_OUTPUT_REQUIRE_SYNC_PASS:-0}",
|
|
"LESAVKA_OUTPUT_DELAY_TARGET=${LESAVKA_OUTPUT_DELAY_TARGET:-video}",
|
|
"LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${LESAVKA_OUTPUT_DELAY_MIN_PAIRS:-13}",
|
|
"LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS:-5000}",
|
|
"LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS=${LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS:-80}",
|
|
"LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_OUTPUT_DELAY_MAX_STEP_US:-1500000}",
|
|
"LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS:-1000}",
|
|
"LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS:-100}",
|
|
"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}",
|
|
"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS:-0}",
|
|
"REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS:-1}",
|
|
"remote_capture_ready_settle_seconds=${14}",
|
|
"announce_capture_start",
|
|
"signal_capture_ready",
|
|
"run_tolerant_capture",
|
|
"discarding %ss of post-enumeration capture before probe",
|
|
"ffmpeg -nostdin -hide_banner",
|
|
"\"$@\" </dev/null",
|
|
"timeout --kill-after=5 --signal=INT",
|
|
"timeout --kill-after=3 --signal=INT",
|
|
"REMOTE_EXPECT_UVC_WIDTH=${REMOTE_EXPECT_UVC_WIDTH:-}",
|
|
"REMOTE_EXPECT_UVC_HEIGHT=${REMOTE_EXPECT_UVC_HEIGHT:-}",
|
|
"REMOTE_EXPECT_UVC_FPS=${REMOTE_EXPECT_UVC_FPS:-}",
|
|
"server.env UVC_MODE=",
|
|
"uvc.env UVC_MODE=",
|
|
"effective UVC_MODE=",
|
|
"expected effective UVC_WIDTH",
|
|
"expected uvc.env UVC_FPS",
|
|
"server-to-capture clock alignment unavailable; falling back to client-mediated SSH samples",
|
|
"LESAVKA_CLOCK_ALIGNMENT_SAMPLES=${LESAVKA_CLOCK_ALIGNMENT_SAMPLES:-5}",
|
|
"sample_best_host_clock_offset_ns",
|
|
"sample_server_to_capture_clock_offset_ns",
|
|
"persistent-ssh-python",
|
|
"server-to-capture-persistent-ssh-python",
|
|
"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}",
|
|
"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US:-0}",
|
|
"write_clock_alignment",
|
|
"capture_start_unix_ns=",
|
|
"write_output_delay_calibration",
|
|
"extract_server_timeline",
|
|
"write_output_delay_correlation",
|
|
"maybe_apply_output_delay_calibration",
|
|
"maybe_run_output_delay_confirmation",
|
|
"enforce_sync_verdict",
|
|
"schema\": \"lesavka.output-delay-calibration.v1\"",
|
|
"schema\": \"lesavka.output-delay-correlation.v1\"",
|
|
"schema\": \"lesavka.clock-alignment.v1\"",
|
|
"schema\": \"lesavka.output-freshness-summary.v1\"",
|
|
"server_timeline_json=",
|
|
"clock_alignment_json: ${LOCAL_CLOCK_ALIGNMENT_JSON}",
|
|
"dominant_layer",
|
|
"freshness status",
|
|
"clock-corrected server feed to Tethys capture event",
|
|
"intentional sync delays",
|
|
"media timestamp path offset",
|
|
"RC target event age",
|
|
"freshness budget",
|
|
"Output smoothness",
|
|
"video continuity",
|
|
"audio pilot continuity",
|
|
"schema\": \"lesavka.output-smoothness-summary.v1\"",
|
|
"video_freshness_ms",
|
|
"audio_freshness_ms",
|
|
"video_event_age_ms",
|
|
"audio_event_age_ms",
|
|
"intentional_video_delay_ms",
|
|
"video_event_age_stats",
|
|
"worst_event_age_with_uncertainty_ms",
|
|
"minimum_event_age_ms",
|
|
"sync_passed",
|
|
"sync did not pass, so freshness from paired signatures is not trustworthy",
|
|
"impossible negative RC event age",
|
|
"video_delay_function_candidate",
|
|
"source\": \"direct-uvc-uac-output-probe\"",
|
|
"scope\": \"server-output-static-baseline\"",
|
|
"applies_to\": \"server UVC/UAC gadget output path\"",
|
|
"measurement_host_role\": \"lab-attached USB host\"",
|
|
"probe_media_origin\": \"server-generated\"",
|
|
"probe_media_path\": \"server generated signatures -> UVC/UAC sinks -> lab host capture\"",
|
|
"audio_after_video_positive",
|
|
"active_audio_offset_us",
|
|
"active_video_offset_us",
|
|
"audio_target_offset_us = active_audio_offset_us + audio_delta_us",
|
|
"video_target_offset_us = active_video_offset_us + video_delta_us",
|
|
"output_delay_active_audio_offset_us",
|
|
"output_delay_active_video_offset_us",
|
|
"audio_target_offset_us",
|
|
"video_target_offset_us",
|
|
"output_delay_audio_target_offset_us",
|
|
"output_delay_video_target_offset_us",
|
|
"LESAVKA_OUTPUT_DELAY_CONFIRMING=1",
|
|
"LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=1",
|
|
"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=\"${confirm_audio_delay}\"",
|
|
"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=\"${confirm_video_delay}\"",
|
|
"==> confirming fixed UVC/UAC output-delay calibration",
|
|
"required sync pass failed",
|
|
"calibration_active_video_offset_us",
|
|
"absolute_target_video_offset_us",
|
|
"calibration_apply_video_delta_us",
|
|
"PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}",
|
|
"\"${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US}\"",
|
|
"\"${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}\"",
|
|
"REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}",
|
|
"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}",
|
|
"output_delay_calibration_json",
|
|
"direct UVC/UAC output-delay calibration",
|
|
"calibration-save-default",
|
|
"LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-0}",
|
|
"PROBE_START_GRACE_SECONDS=${PROBE_START_GRACE_SECONDS:-20}",
|
|
"PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + PROBE_START_GRACE_SECONDS))}",
|
|
"CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + PROBE_START_GRACE_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))}",
|
|
"ANALYSIS_TIMELINE_WINDOW=${ANALYSIS_TIMELINE_WINDOW:-0}",
|
|
"compute_analysis_window_arg",
|
|
"analyzer timeline window:",
|
|
"output-delay-probe",
|
|
"server-generated UVC/UAC output-delay probe",
|
|
"server output-delay probe timed out after ${PROBE_TIMEOUT_SECONDS}s",
|
|
"--event-width-codes '${PROBE_EVENT_WIDTH_CODES}'",
|
|
"VIDIOC_STREAMON.*Connection timed out",
|
|
"the UVC host opened before MJPEG frames reached the gadget",
|
|
"Tethys capture failed before the sync probe could start",
|
|
"wait_for_capture_ready",
|
|
"Timed out waiting for Tethys capture to become ready",
|
|
"grep -q \"${CAPTURE_READY_MARKER}\"",
|
|
"Lesavka UVC video device not found on Tethys; refusing to fall back to an unrelated webcam/capture card.",
|
|
"resolve_alsa_audio_device",
|
|
"PipeWire Lesavka source not found; using explicit diagnostic ALSA fallback device",
|
|
"Set REMOTE_CAPTURE_STACK=alsa or REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=1 only for diagnostic signal-presence checks.",
|
|
"Lesavka Pulse/PipeWire audio source not found; refusing raw ALSA fallback for timing-sensitive capture.",
|
|
"discarding %ss of post-enumeration capture before probe",
|
|
"using Pulse source:",
|
|
"-f pulse",
|
|
"-map 1:a:0",
|
|
"artifact_dir: ${LOCAL_REPORT_DIR}",
|
|
"events_csv: ${LOCAL_EVENTS_CSV}",
|
|
"server_timeline_json: ${LOCAL_SERVER_TIMELINE_JSON}",
|
|
"output_delay_correlation_json: ${LOCAL_OUTPUT_DELAY_CORRELATION_JSON}",
|
|
"==> Lesavka versions under test",
|
|
"lesavka-relayctl",
|
|
"--bin lesavka-relayctl",
|
|
"--bin lesavka-sync-analyze",
|
|
"client_revision=",
|
|
"server_version=",
|
|
"server_revision=",
|
|
"combined version+revision",
|
|
] {
|
|
assert!(
|
|
SYNC_SCRIPT.contains(expected),
|
|
"manual sync script should contain {expected}"
|
|
);
|
|
}
|
|
assert!(
|
|
!SYNC_SCRIPT.contains(
|
|
"RESOLVED_LESAVKA_SERVER_ADDR=\"http://${LESAVKA_SERVER_CONNECT_HOST}:${port}\""
|
|
),
|
|
"auto server resolution should not guess a public gRPC host when SSH is already required"
|
|
);
|
|
for forbidden in [
|
|
"LOCAL_AUDIO_SANITY",
|
|
"run_local_audio_sanity.sh",
|
|
"lesavka-sync-probe",
|
|
] {
|
|
assert!(
|
|
!SYNC_SCRIPT.contains(forbidden),
|
|
"direct UVC/UAC output-delay probe must not run workstation/client-side probe behavior: {forbidden}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn server_rc_mode_matrix_validates_advertised_uvc_profiles() {
|
|
for expected in [
|
|
"LESAVKA_SERVER_RC_CORE_WEBCAM_MODES=${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES:-1280x720@20,1280x720@30,1920x1080@20,1920x1080@30}",
|
|
"LESAVKA_SERVER_RC_MODES=${LESAVKA_SERVER_RC_MODES:-${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES}}",
|
|
"LESAVKA_SERVER_REPO=${LESAVKA_SERVER_REPO:-auto}",
|
|
"LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US:-${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}}",
|
|
"LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US:-170000}",
|
|
"LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US:-1280x720@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1280x720@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US}}",
|
|
"LESAVKA_SERVER_RC_MODE_DELAYS_US=${LESAVKA_SERVER_RC_MODE_DELAYS_US:-1280x720@20=170000,1280x720@30=170000,1920x1080@20=170000,1920x1080@30=170000}",
|
|
"LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES=${LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES:-1280x720,1920x1080}",
|
|
"LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS=${LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS:-20,30}",
|
|
"LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX:-Logitech|BRIO|C9[0-9]+|HD UVC WebCam|USB2[.]0 HD|Integrated Camera|Webcam|Camera}",
|
|
"LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX:-Lesavka|UGREEN|MACROSILICON|Composite|Capture}",
|
|
"LESAVKA_SERVER_RC_MODE_SOURCE=${LESAVKA_SERVER_RC_MODE_SOURCE:-configured}",
|
|
"LESAVKA_SERVER_RC_RECONFIGURE=${LESAVKA_SERVER_RC_RECONFIGURE:-0}",
|
|
"LESAVKA_SERVER_RC_RECONFIGURE_UPDATE=${LESAVKA_SERVER_RC_RECONFIGURE_UPDATE:-0}",
|
|
"LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY:-runtime}",
|
|
"LESAVKA_SERVER_RC_ALLOW_GADGET_RESET=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET:-1}",
|
|
"LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE=${LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE:-0}",
|
|
"LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY=${LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY:-1}",
|
|
"LESAVKA_SERVER_RC_WAIT_TETHYS_READY=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY:-1}",
|
|
"LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS:-60}",
|
|
"LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS:-6}",
|
|
"LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS:-3}",
|
|
"LESAVKA_SERVER_RC_PROBE_PREBUILD=${LESAVKA_SERVER_RC_PROBE_PREBUILD:-1}",
|
|
"LESAVKA_SERVER_RC_TUNE_DELAYS=${LESAVKA_SERVER_RC_TUNE_DELAYS:-1}",
|
|
"LESAVKA_SERVER_RC_TUNE_CONFIRM=${LESAVKA_SERVER_RC_TUNE_CONFIRM:-1}",
|
|
"LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-13}",
|
|
"LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS:-1000}",
|
|
"LESAVKA_SERVER_RC_TUNE_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US:-500000}",
|
|
"LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000}",
|
|
"Theia sudo password for %s",
|
|
"==> priming remote sudo on ${LESAVKA_SERVER_HOST}",
|
|
"==> prebuilding relay control/analyzer once for the mode matrix",
|
|
"LESAVKA_SERVER_RC_MODES=auto",
|
|
"discover_local_webcam_modes",
|
|
"lookup_audio_delay_us",
|
|
"local webcam",
|
|
"mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}",
|
|
"video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}",
|
|
"audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}",
|
|
"pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL}",
|
|
"fast runtime env updated: CAM_OUTPUT=uvc",
|
|
"cycling UVC gadget descriptors",
|
|
"lesavka-core reconfigure log:",
|
|
"missing /usr/local/bin/lesavka-core.sh",
|
|
"wait_tethys_media_ready",
|
|
"==> waiting for Tethys media endpoints for ${mode}",
|
|
"Tethys media ready: video=%s mode=%s audio_stack=%s",
|
|
"timed out waiting for Tethys Lesavka media endpoints",
|
|
"LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350}",
|
|
"LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS=${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS:-0}",
|
|
"LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS=${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS:-0}",
|
|
"LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES:-12}",
|
|
"mode-matrix-summary.json",
|
|
"mode-matrix-summary.csv",
|
|
"mode-matrix-summary.txt",
|
|
"mode-delay-recommendations.json",
|
|
"mode-delay-recommendations.env",
|
|
"schema\": \"lesavka.server-rc-mode-result.v1\"",
|
|
"schema\": \"lesavka.server-rc-mode-matrix-summary.v1\"",
|
|
"schema\": \"lesavka.server-rc-mode-delay-recommendations.v1\"",
|
|
"output_delay_calibration",
|
|
"write_tune_candidate_env",
|
|
"mode-result-seed.json",
|
|
"mode-result-tuned.json",
|
|
"==> mode ${mode}: confirming tuned delays",
|
|
"calibration_ready",
|
|
"calibration_video_target_offset_us",
|
|
"calibration_audio_target_offset_us",
|
|
"calibration:",
|
|
"signature_coverage",
|
|
"paired coded signatures",
|
|
"signature_missing_codes",
|
|
"REMOTE_PULSE_CAPTURE_TOOL=\"${REMOTE_PULSE_CAPTURE_TOOL}\"",
|
|
"REMOTE_PULSE_VIDEO_MODE=\"${REMOTE_PULSE_VIDEO_MODE}\"",
|
|
"REMOTE_CAPTURE_STACK=\"${REMOTE_CAPTURE_STACK}\"",
|
|
"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=\"${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}\"",
|
|
"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=\"${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}\"",
|
|
"REMOTE_CAPTURE_READY_SETTLE_SECONDS=\"${REMOTE_CAPTURE_READY_SETTLE_SECONDS}\"",
|
|
"PROBE_PREBUILD=0",
|
|
"VIDEO_SIZE=\"${width}x${height}\"",
|
|
"VIDEO_FPS=\"${fps}\"",
|
|
"REMOTE_EXPECT_UVC_WIDTH=\"${width}\"",
|
|
"REMOTE_EXPECT_UVC_HEIGHT=\"${height}\"",
|
|
"REMOTE_EXPECT_UVC_FPS=\"${fps}\"",
|
|
"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=\"${audio_delay_us}\"",
|
|
"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=\"${video_delay_us}\"",
|
|
"LESAVKA_OUTPUT_DELAY_APPLY=0",
|
|
"LESAVKA_OUTPUT_DELAY_SAVE=0",
|
|
"LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=\"${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}\"",
|
|
"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=\"${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}\"",
|
|
"sync did not pass",
|
|
"freshness did not pass",
|
|
"video hiccups",
|
|
"estimated missing video frames",
|
|
"audio hiccups",
|
|
] {
|
|
assert!(
|
|
SERVER_RC_MODE_MATRIX_SCRIPT.contains(expected),
|
|
"server-to-RC mode matrix script should contain {expected}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn browser_sync_script_can_delegate_to_a_real_path_driver() {
|
|
for expected in [
|
|
"BROWSER_RECORD_SECONDS=${BROWSER_RECORD_SECONDS:-${PROBE_DURATION_SECONDS}}",
|
|
"BROWSER_SYNC_DRIVER_COMMAND=${BROWSER_SYNC_DRIVER_COMMAND:-}",
|
|
"BROWSER_CONSUMER_REUSE_SESSION=${BROWSER_CONSUMER_REUSE_SESSION:-0}",
|
|
"BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED:-1}",
|
|
"SYNC_ANALYZE_EVENT_WIDTH_CODES=${SYNC_ANALYZE_EVENT_WIDTH_CODES:-}",
|
|
"==> running custom browser sync driver",
|
|
"bash -lc \"${BROWSER_SYNC_DRIVER_COMMAND}\"",
|
|
"browser_start_token=${browser_start_token}",
|
|
"uploaded_start_token",
|
|
"BROWSER_START_TOKEN",
|
|
"analysis-failure.json",
|
|
"BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED}",
|
|
"BROWSER_START_ATTEMPTS=${BROWSER_START_ATTEMPTS:-5}",
|
|
"browser consumer start attempt ${attempt}/${BROWSER_START_ATTEMPTS} failed; retrying",
|
|
"--event-width-codes",
|
|
"--report-dir \"${LOCAL_REPORT_DIR}\"",
|
|
"for attempt in 1 2 3 4 5",
|
|
"capture fetch attempt ${attempt} failed; retrying",
|
|
"failed to fetch browser capture from ${TETHYS_HOST}:${REMOTE_CAPTURE}",
|
|
r"raw activity delta was ([+-]?[0-9]+(?:\.[0-9]+)?) ms ",
|
|
r"\(video=([0-9]+(?:\.[0-9]+)?)s audio=([0-9]+(?:\.[0-9]+)?)s\)",
|
|
] {
|
|
assert!(
|
|
BROWSER_SYNC_SCRIPT.contains(expected),
|
|
"browser sync script should contain {expected}"
|
|
);
|
|
}
|
|
assert!(
|
|
!BROWSER_SYNC_SCRIPT.contains(r"(?:\\.[0-9]+)?"),
|
|
"browser sync raw-delta parser should not require a literal backslash before decimals"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn sync_probe_runner_uses_bundled_webcam_media_path() {
|
|
for expected in [
|
|
"bundled_webcam_media",
|
|
"refusing to measure split upstream",
|
|
"UpstreamMediaBundle",
|
|
"stream_webcam_media",
|
|
"PROBE_BUNDLE_SESSION_ID",
|
|
"PROBE_BUNDLE_AUDIO_GRACE",
|
|
"server does not advertise bundled webcam media",
|
|
] {
|
|
assert!(
|
|
SYNC_PROBE_RUNNER.contains(expected),
|
|
"sync probe runner should contain {expected}"
|
|
);
|
|
}
|
|
for forbidden in [
|
|
".stream_camera(Request::new(outbound))",
|
|
".stream_microphone(Request::new(outbound))",
|
|
] {
|
|
assert!(
|
|
!SYNC_PROBE_RUNNER.contains(forbidden),
|
|
"sync probe runner must not use old split upstream RPC {forbidden}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn mirrored_sync_script_uses_real_client_capture_path() {
|
|
for expected in [
|
|
"local_av_stimulus.py",
|
|
"lesavka-client",
|
|
"LESAVKA_HEADLESS=1",
|
|
"LESAVKA_MEDIA_CONTROL=\"${MEDIA_CONTROL}\"",
|
|
"LESAVKA_REQUIRE_EXPLICIT_MEDIA_SOURCES=\"${LESAVKA_REQUIRE_EXPLICIT_MEDIA_SOURCES:-1}\"",
|
|
"--no-launcher --server \"${RESOLVED_LESAVKA_SERVER_ADDR}\"",
|
|
"BROWSER_SYNC_DRIVER_COMMAND=\"${driver_command}\"",
|
|
"SYNC_ANALYZE_EVENT_WIDTH_CODES=\"${PROBE_EVENT_WIDTH_CODES}\"",
|
|
"run_upstream_browser_av_sync.sh",
|
|
"wait_for_stimulus_page_ready 15",
|
|
"Point the real webcam at the stimulus window",
|
|
"==> Lesavka versions under test",
|
|
"lesavka-relayctl",
|
|
"--bin lesavka-relayctl",
|
|
"client_revision=",
|
|
"server_version=",
|
|
"server_revision=",
|
|
"combined version+revision",
|
|
"run_status=0",
|
|
"run_mirrored_segments || run_status=$?",
|
|
"LESAVKA_SYNC_APPLY_CALIBRATION=${LESAVKA_SYNC_APPLY_CALIBRATION:-0}",
|
|
"LESAVKA_SYNC_SAVE_CALIBRATION=${LESAVKA_SYNC_SAVE_CALIBRATION:-0}",
|
|
"LESAVKA_SYNC_CALIBRATION_TARGET=${LESAVKA_SYNC_CALIBRATION_TARGET:-video}",
|
|
"LESAVKA_SYNC_ADAPTIVE_CALIBRATION=${LESAVKA_SYNC_ADAPTIVE_CALIBRATION:-0}",
|
|
"LESAVKA_SYNC_CALIBRATION_SEGMENTS=${LESAVKA_SYNC_CALIBRATION_SEGMENTS:-1}",
|
|
"LESAVKA_SYNC_CONTINUOUS_BROWSER=${LESAVKA_SYNC_CONTINUOUS_BROWSER:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
|
|
"LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE=${LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
|
|
"LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS=${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS:-3}",
|
|
"PROBE_AUDIO_GAIN=${PROBE_AUDIO_GAIN:-0.55}",
|
|
"LESAVKA_STIMULUS_PREVIEW_SECONDS=${LESAVKA_STIMULUS_PREVIEW_SECONDS:-4}",
|
|
"LESAVKA_OPEN_MANUAL_REVIEW_DOLPHIN=${LESAVKA_OPEN_MANUAL_REVIEW_DOLPHIN:-1}",
|
|
"LESAVKA_SYNC_PROVISIONAL_CALIBRATION=${LESAVKA_SYNC_PROVISIONAL_CALIBRATION:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
|
|
"LESAVKA_SYNC_PROVISIONAL_MIN_PAIRS=${LESAVKA_SYNC_PROVISIONAL_MIN_PAIRS:-3}",
|
|
"LESAVKA_SYNC_PROVISIONAL_MAX_P95_MS=${LESAVKA_SYNC_PROVISIONAL_MAX_P95_MS:-350}",
|
|
"LESAVKA_SYNC_PROVISIONAL_MAX_DRIFT_MS=${LESAVKA_SYNC_PROVISIONAL_MAX_DRIFT_MS:-250}",
|
|
"LESAVKA_SYNC_PROVISIONAL_GAIN=${LESAVKA_SYNC_PROVISIONAL_GAIN:-0.5}",
|
|
"LESAVKA_SYNC_PROVISIONAL_MAX_STEP_US=${LESAVKA_SYNC_PROVISIONAL_MAX_STEP_US:-150000}",
|
|
"LESAVKA_SYNC_RAW_FAILURE_CALIBRATION=${LESAVKA_SYNC_RAW_FAILURE_CALIBRATION:-0}",
|
|
"LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS=${LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS:-3}",
|
|
"LESAVKA_SYNC_RAW_FAILURE_MAX_ABS_DELTA_MS=${LESAVKA_SYNC_RAW_FAILURE_MAX_ABS_DELTA_MS:-350}",
|
|
"LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION=${LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
|
|
"LESAVKA_SYNC_CONFIRMATION_SEGMENTS=${LESAVKA_SYNC_CONFIRMATION_SEGMENTS:-1}",
|
|
"LESAVKA_SYNC_REQUIRE_CONFIRMATION_PASS=${LESAVKA_SYNC_REQUIRE_CONFIRMATION_PASS:-${LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION}}",
|
|
"LESAVKA_SYNC_CONFIRMATION_SEGMENTS must be a non-negative integer",
|
|
"LESAVKA_SYNC_TOTAL_SEGMENTS=$((LESAVKA_SYNC_CALIBRATION_SEGMENTS + LESAVKA_SYNC_CONFIRMATION_SEGMENTS))",
|
|
"export LESAVKA_SYNC_PROVISIONAL_CALIBRATION",
|
|
"export LESAVKA_SYNC_RAW_FAILURE_CALIBRATION",
|
|
"export LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS",
|
|
"LESAVKA_SYNC_ADAPTIVE_CALIBRATION",
|
|
"LESAVKA_SYNC_CALIBRATION_SEGMENTS=4",
|
|
"browser_consumer_reuse_session=${reuse_browser_session}",
|
|
"browser_analysis_required=${analysis_required}",
|
|
"BROWSER_CONSUMER_REUSE_SESSION=\"${reuse_browser_session}\"",
|
|
"BROWSER_ANALYSIS_REQUIRED=\"${analysis_required}\"",
|
|
"--audio-gain \"${PROBE_AUDIO_GAIN}\"",
|
|
"run_stimulus_preview",
|
|
"write_stimulus_driver_script",
|
|
"local_stimulus_started=true",
|
|
"observed_start_token",
|
|
"audio_state",
|
|
"LESAVKA_SYNC_CALIBRATION_SEGMENTS must be a positive integer",
|
|
"run_mirrored_segments",
|
|
"summarize_adaptive_probe_metrics",
|
|
"for segment in $(seq 1 \"${LESAVKA_SYNC_TOTAL_SEGMENTS}\")",
|
|
"segment_phase",
|
|
"confirmation segment: calibration apply disabled so this segment tests the active calibration",
|
|
"segment-${segment}",
|
|
"calibration-before.env",
|
|
"planner-before.env",
|
|
"calibration-decision.env",
|
|
"segment-metrics.csv",
|
|
"segment-metrics.jsonl",
|
|
"segment-events.csv",
|
|
"segment-events.jsonl",
|
|
"manual-review",
|
|
"manual_review_html",
|
|
"manual_review_dir",
|
|
"open_manual_review_in_dolphin",
|
|
"dolphin",
|
|
"capture_path",
|
|
"confirmation-summary.json",
|
|
"confirmation_passed",
|
|
"check_confirmation_result",
|
|
"confirmation check failed",
|
|
"analysis_failure_reason",
|
|
"probe_activity_start_delta_ms",
|
|
"blind-targets.json",
|
|
"no segment produced a passing probe verdict; refusing to invent blind targets",
|
|
"confirmation did not pass; refusing to promote calibration-only segments to blind targets",
|
|
"candidate_good_calibration_segments",
|
|
"decision_mode",
|
|
"decision_provisional_video_recommendation_us",
|
|
"planner_live_lag_ms_after",
|
|
"probe_p95_abs_skew_ms",
|
|
"transport/server receive jitter",
|
|
"settling ${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS}s before next segment",
|
|
"print_upstream_calibration_state \"before mirrored run\"",
|
|
"maybe_apply_probe_calibration",
|
|
"calibration_ready=${calibration_ready}",
|
|
"calibration_decision_mode=${calibration_decision_mode}",
|
|
"bounded provisional correction from median skew",
|
|
"bounded provisional correction from analyzer-failure raw activity",
|
|
"raw_failure_calibration_enabled",
|
|
"raw analyzer-failure calibration refused: ",
|
|
"raw_failure_min_pairs",
|
|
"provisional calibration not saved",
|
|
"calibration apply refused: ${calibration_decision_note}",
|
|
"calibrate \"${calibration_apply_audio_delta_us}\" \"${calibration_apply_video_delta_us}\"",
|
|
"LESAVKA_UPSTREAM_BLIND_HEAL is server-side",
|
|
"calibration-save-default",
|
|
"print_upstream_sync_state \"after mirrored run\"",
|
|
"print_upstream_calibration_state \"after mirrored run\"",
|
|
"==> mirrored probe failed",
|
|
] {
|
|
assert!(
|
|
MIRRORED_SYNC_SCRIPT.contains(expected),
|
|
"mirrored sync script should contain {expected}"
|
|
);
|
|
}
|
|
assert!(
|
|
!MIRRORED_SYNC_SCRIPT.contains("lesavka-sync-probe"),
|
|
"mirrored sync must not use the synthetic direct sender"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn local_stimulus_matches_sync_analyzer_pulse_contract() {
|
|
for expected in [
|
|
"--warmup-seconds",
|
|
"--pulse-period-ms",
|
|
"--pulse-width-ms",
|
|
"--marker-tick-period",
|
|
"--audio-gain",
|
|
"/preview",
|
|
"preview_token",
|
|
"observed_preview_token",
|
|
"completed_preview_token",
|
|
"stimulus preview running",
|
|
"stimulus preview completed",
|
|
"audio_state",
|
|
"--event-width-codes",
|
|
"event_width_codes",
|
|
"audio_gain",
|
|
"widthCode",
|
|
"oscillator.frequency.value = 880",
|
|
"setStatus(`ready",
|
|
"Point the real webcam at this window",
|
|
] {
|
|
assert!(
|
|
LOCAL_STIMULUS.contains(expected),
|
|
"local stimulus should contain {expected}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn manual_probe_python_servers_use_reentrant_state_locks() {
|
|
const BROWSER_CONSUMER: &str = include_str!("../../scripts/manual/browser_consumer_probe.py");
|
|
for (name, script) in [
|
|
("local stimulus", LOCAL_STIMULUS),
|
|
("browser consumer", BROWSER_CONSUMER),
|
|
] {
|
|
assert!(
|
|
script.contains("threading.RLock()"),
|
|
"{name} server request handlers call snapshot while holding state lock; use RLock to avoid /start deadlocks"
|
|
);
|
|
}
|
|
}
|