lesavka/testing/tests/client_manual_sync_script_contract.rs

690 lines
34 KiB
Rust

//! Contract tests for the manual upstream A/V sync harness.
//!
//! Scope: statically guard the workstation-side tunnel/bootstrap behavior.
//! Targets: `scripts/manual/run_upstream_av_sync.sh`.
//! Why: the manual probe should reach Theia through SSH even when the gRPC
//! port is not exposed on the public SSH endpoint.
const SYNC_SCRIPT: &str = include_str!("../../scripts/manual/run_upstream_av_sync.sh");
const SERVER_RC_MODE_MATRIX_SCRIPT: &str =
include_str!("../../scripts/manual/run_server_to_rc_mode_matrix.sh");
const BROWSER_SYNC_SCRIPT: &str =
include_str!("../../scripts/manual/run_upstream_browser_av_sync.sh");
const MIRRORED_SYNC_SCRIPT: &str =
include_str!("../../scripts/manual/run_upstream_mirrored_av_sync.sh");
const LOCAL_STIMULUS: &str = include_str!("../../scripts/manual/local_av_stimulus.py");
const SYNC_PROBE_RUNNER: &str = include_str!("../../client/src/sync_probe/runner.rs");
#[test]
fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
for expected in [
"cleanup_server_tunnel",
"pick_local_server_tunnel_port",
"wait_for_server_tunnel",
"start_server_tunnel",
"ExitOnForwardFailure=yes",
"127.0.0.1:${local_port}:127.0.0.1:${remote_port}",
"LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https}",
"LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}",
"RESOLVED_LESAVKA_SERVER_ADDR=\"${LESAVKA_SERVER_SCHEME}://127.0.0.1:${SERVER_TUNNEL_LOCAL_PORT}\"",
"LESAVKA_TLS_DOMAIN=\"${LESAVKA_TLS_DOMAIN}\"",
"tunneled to ${LESAVKA_SERVER_HOST}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT}",
"CAPTURE_READY_MARKER=\"__LESAVKA_CAPTURE_READY__\"",
"LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}",
"LOCAL_REPORT_DIR=\"${LOCAL_OUTPUT_DIR%/}/lesavka-output-delay-probe-${STAMP}\"",
"LOCAL_ANALYSIS_JSON=\"${LOCAL_REPORT_DIR}/report.json\"",
"LOCAL_EVENTS_CSV=\"${LOCAL_REPORT_DIR}/events.csv\"",
"LOCAL_SERVER_PROBE_REPLY=\"${LOCAL_REPORT_DIR}/server-output-probe-reply.txt\"",
"LOCAL_SERVER_TIMELINE_JSON=\"${LOCAL_REPORT_DIR}/server-output-timeline.json\"",
"LOCAL_OUTPUT_DELAY_CORRELATION_JSON=\"${LOCAL_REPORT_DIR}/output-delay-correlation.json\"",
"LOCAL_OUTPUT_DELAY_CORRELATION_CSV=\"${LOCAL_REPORT_DIR}/output-delay-correlation.csv\"",
"LOCAL_OUTPUT_DELAY_CORRELATION_TXT=\"${LOCAL_REPORT_DIR}/output-delay-correlation.txt\"",
"LOCAL_CLOCK_ALIGNMENT_JSON=\"${LOCAL_REPORT_DIR}/clock-alignment.json\"",
"LOCAL_OUTPUT_DELAY_JSON=\"${LOCAL_REPORT_DIR}/output-delay-calibration.json\"",
"LOCAL_OUTPUT_DELAY_ENV=\"${LOCAL_REPORT_DIR}/output-delay-calibration.env\"",
"LESAVKA_OUTPUT_DELAY_CALIBRATION=${LESAVKA_OUTPUT_DELAY_CALIBRATION:-1}",
"LESAVKA_OUTPUT_DELAY_APPLY=${LESAVKA_OUTPUT_DELAY_APPLY:-0}",
"LESAVKA_OUTPUT_DELAY_APPLY_MODE=${LESAVKA_OUTPUT_DELAY_APPLY_MODE:-absolute}",
"LESAVKA_OUTPUT_DELAY_CONFIRM=${LESAVKA_OUTPUT_DELAY_CONFIRM:-1}",
"LESAVKA_OUTPUT_DELAY_SAVE=${LESAVKA_OUTPUT_DELAY_SAVE:-0}",
"LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=${LESAVKA_OUTPUT_REQUIRE_SYNC_PASS:-0}",
"LESAVKA_OUTPUT_DELAY_TARGET=${LESAVKA_OUTPUT_DELAY_TARGET:-video}",
"LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${LESAVKA_OUTPUT_DELAY_MIN_PAIRS:-13}",
"LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS:-5000}",
"LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS=${LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS:-80}",
"LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_OUTPUT_DELAY_MAX_STEP_US:-1500000}",
"LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS:-1000}",
"LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS:-100}",
"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}",
"LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=${LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS:-${LESAVKA_OUTPUT_DELAY_MIN_PAIRS}}",
"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS:-0}",
"REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS:-1}",
"remote_capture_ready_settle_seconds=${15}",
"announce_capture_start",
"signal_capture_ready",
"run_tolerant_capture",
"discarding %ss of post-enumeration capture before probe",
"ffmpeg -nostdin -hide_banner",
"\"$@\" </dev/null",
"timeout --kill-after=5 --signal=INT",
"timeout --kill-after=3 --signal=INT",
"REMOTE_EXPECT_UVC_WIDTH=${REMOTE_EXPECT_UVC_WIDTH:-}",
"REMOTE_EXPECT_UVC_HEIGHT=${REMOTE_EXPECT_UVC_HEIGHT:-}",
"REMOTE_EXPECT_UVC_FPS=${REMOTE_EXPECT_UVC_FPS:-}",
"server.env UVC_MODE=",
"uvc.env UVC_MODE=",
"effective UVC_MODE=",
"expected effective UVC_WIDTH",
"expected uvc.env UVC_FPS",
"server-to-capture clock alignment unavailable; falling back to client-mediated SSH samples",
"LESAVKA_CLOCK_ALIGNMENT_SAMPLES=${LESAVKA_CLOCK_ALIGNMENT_SAMPLES:-5}",
"sample_best_host_clock_offset_ns",
"sample_server_to_capture_clock_offset_ns",
"persistent-ssh-python",
"server-to-capture-persistent-ssh-python",
"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}",
"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US:-0}",
"write_clock_alignment",
"capture_start_unix_ns=",
"write_output_delay_calibration",
"extract_server_timeline",
"write_output_delay_correlation",
"maybe_apply_output_delay_calibration",
"maybe_run_output_delay_confirmation",
"enforce_sync_verdict",
"schema\": \"lesavka.output-delay-calibration.v1\"",
"schema\": \"lesavka.output-delay-correlation.v1\"",
"schema\": \"lesavka.clock-alignment.v1\"",
"schema\": \"lesavka.output-freshness-summary.v1\"",
"schema\": \"lesavka.output-timing-map.v1\"",
"server_timeline_json=",
"clock_alignment_json: ${LOCAL_CLOCK_ALIGNMENT_JSON}",
"dominant_layer",
"Timing map",
"capture to server probe start",
"setup/pre-roll, not media latency",
"injection point",
"server-final-output-handoff",
"video CameraRelay::feed; audio Voice::push",
"client uplink included",
"freshness clock starts",
"freshness status",
"clock-corrected server pipeline reference to Tethys observed playback",
"capture timebase",
"capture timebase invalid",
"capture_timebase_warnings = []",
"capture timebase warning",
"treating whole-file media span as recorder coverage, not event freshness",
"intentional sync delays",
"sink handoff overhead",
"pipeline freshness",
"Freshness timeline",
"capture duration",
"observed_capture_duration_s",
"configured_capture_duration_s",
"recorder coverage, not media latency",
"active probe media",
"setup before probe",
"measured event freshness",
"freshness verdict",
"freshness budget",
"Output smoothness",
"capture media timeline",
"media_timeline",
"video continuity",
"audio pilot continuity",
"schema\": \"lesavka.output-smoothness-summary.v1\"",
"video_freshness_ms",
"audio_freshness_ms",
"video_event_age_ms",
"audio_event_age_ms",
"intentional_video_delay_ms",
"video_event_age_stats",
"video_sink_handoff_overhead_stats",
"minimum_pipeline_freshness_ms",
"worst_event_age_with_uncertainty_ms",
"minimum_event_age_ms",
"min_paired_events",
"paired_event_count",
"non_monotonic_event_timestamps",
"sync_passed",
"sync did not pass, so freshness from paired signatures is not trustworthy",
"impossible negative pipeline freshness",
"video_delay_function_candidate",
"source\": \"direct-uvc-uac-output-probe\"",
"scope\": \"server-output-static-baseline\"",
"applies_to\": \"server UVC/UAC gadget output path\"",
"measurement_host_role\": \"lab-attached USB host\"",
"injection_scope\": \"server-final-output-handoff\"",
"client_uplink_included\": False",
"probe_media_origin\": \"server-generated\"",
"probe_media_path\": \"server generated signatures -> UVC/UAC sinks -> lab host capture\"",
"audio_after_video_positive",
"active_audio_offset_us",
"active_video_offset_us",
"audio_target_offset_us = active_audio_offset_us + audio_delta_us",
"video_target_offset_us = active_video_offset_us + video_delta_us",
"output_delay_active_audio_offset_us",
"output_delay_active_video_offset_us",
"audio_target_offset_us",
"video_target_offset_us",
"output_delay_audio_target_offset_us",
"output_delay_video_target_offset_us",
"LESAVKA_OUTPUT_DELAY_CONFIRMING=1",
"LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=1",
"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=\"${confirm_audio_delay}\"",
"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=\"${confirm_video_delay}\"",
"==> confirming fixed UVC/UAC output-delay calibration",
"required sync pass failed",
"calibration_active_video_offset_us",
"absolute_target_video_offset_us",
"calibration_apply_video_delta_us",
"PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}",
"\"${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US}\"",
"\"${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}\"",
"REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}",
"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}",
"REMOTE_PULSE_AUDIO_ANCHOR_SILENCE=${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE:-1}",
"anchoring Pulse capture audio timeline with generated silence",
"audiotestsrc wave=silence is-live=true do-timestamp=true",
"audiomixer name=amix ignore-inactive-pads=true",
"output_delay_calibration_json",
"direct UVC/UAC output-delay calibration",
"calibration-save-default",
"LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-0}",
"PROBE_START_GRACE_SECONDS=${PROBE_START_GRACE_SECONDS:-20}",
"PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + PROBE_START_GRACE_SECONDS))}",
"CAPTURE_START_GRACE_SECONDS=${CAPTURE_START_GRACE_SECONDS:-12}",
"CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + CAPTURE_START_GRACE_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))}",
"ANALYSIS_TIMELINE_WINDOW=${ANALYSIS_TIMELINE_WINDOW:-0}",
"compute_analysis_window_arg",
"analyzer timeline window:",
"output-delay-probe",
"server-generated UVC/UAC output-delay probe",
"server output-delay probe timed out after ${PROBE_TIMEOUT_SECONDS}s",
"--event-width-codes '${PROBE_EVENT_WIDTH_CODES}'",
"VIDIOC_STREAMON.*Connection timed out",
"the UVC host opened before MJPEG frames reached the gadget",
"Tethys capture failed before the sync probe could start",
"wait_for_capture_ready",
"Timed out waiting for Tethys capture to become ready",
"grep -q \"${CAPTURE_READY_MARKER}\"",
"Lesavka UVC video device not found on Tethys; refusing to fall back to an unrelated webcam/capture card.",
"resolve_alsa_audio_device",
"PipeWire Lesavka source not found; using explicit diagnostic ALSA fallback device",
"Set REMOTE_CAPTURE_STACK=alsa or REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=1 only for diagnostic signal-presence checks.",
"Lesavka Pulse/PipeWire audio source not found; refusing raw ALSA fallback for timing-sensitive capture.",
"discarding %ss of post-enumeration capture before probe",
"using Pulse source:",
"-f pulse",
"-map 1:a:0",
"artifact_dir: ${LOCAL_REPORT_DIR}",
"events_csv: ${LOCAL_EVENTS_CSV}",
"server_timeline_json: ${LOCAL_SERVER_TIMELINE_JSON}",
"output_delay_correlation_json: ${LOCAL_OUTPUT_DELAY_CORRELATION_JSON}",
"==> Lesavka versions under test",
"lesavka-relayctl",
"--bin lesavka-relayctl",
"--bin lesavka-sync-analyze",
"client_revision=",
"server_version=",
"server_revision=",
"combined version+revision",
] {
assert!(
SYNC_SCRIPT.contains(expected),
"manual sync script should contain {expected}"
);
}
assert!(
!SYNC_SCRIPT.contains(
"RESOLVED_LESAVKA_SERVER_ADDR=\"http://${LESAVKA_SERVER_CONNECT_HOST}:${port}\""
),
"auto server resolution should not guess a public gRPC host when SSH is already required"
);
for forbidden in [
"LOCAL_AUDIO_SANITY",
"run_local_audio_sanity.sh",
"lesavka-sync-probe",
] {
assert!(
!SYNC_SCRIPT.contains(forbidden),
"direct UVC/UAC output-delay probe must not run workstation/client-side probe behavior: {forbidden}"
);
}
}
#[test]
fn output_freshness_is_event_level_not_capture_container_duration() {
for expected in [
"\"freshness_clock_starts_at\": \"server pipeline reference before intentional sync delay\"",
"video_freshness_ms = (",
"audio_freshness_ms = (",
"\"video_freshness_stats\": video_freshness_stats",
"\"audio_freshness_stats\": audio_freshness_stats",
"\"worst_p95_pipeline_freshness_ms\": freshness_worst_p95_ms",
"\"worst_event_age_with_uncertainty_ms\": freshness_worst_event_with_uncertainty_ms",
"\"configured_capture_duration_s\": configured_capture_duration_s",
"\"observed_capture_duration_s\": observed_capture_duration_s",
"\"min_paired_events\": min_freshness_pairs",
"\"paired_event_count\": len(joined)",
"\"warnings\": capture_timebase_warnings",
"treating whole-file media span as recorder coverage, not event freshness",
"Freshness timeline",
"measured event freshness",
] {
assert!(
SYNC_SCRIPT.contains(expected),
"freshness contract should contain event-level timing marker {expected}"
);
}
let span_gap_start = SYNC_SCRIPT
.find("span_gap_s = finite(media_timeline.get(\"audio_video_span_gap_s\"))")
.expect("manual sync script should compute media span gap diagnostics");
let span_gap_end = SYNC_SCRIPT[span_gap_start..]
.find("if capture_timebase_reasons:")
.map(|offset| span_gap_start + offset)
.expect("span gap diagnostic block should end before reason folding");
let span_gap_block = &SYNC_SCRIPT[span_gap_start..span_gap_end];
assert!(
span_gap_block.contains("capture_timebase_warnings.append("),
"whole-file media span mismatch should be recorded as a warning"
);
assert!(
!span_gap_block.contains("capture_timebase_status = \"invalid\""),
"whole-file media span mismatch must not invalidate freshness; freshness is event-level"
);
assert!(
!span_gap_block.contains("capture_timebase_reasons.append("),
"whole-file media span mismatch must not become a hard invalidation reason"
);
}
#[test]
fn output_freshness_still_invalidates_real_event_timing_contradictions() {
let negative_rows_start = SYNC_SCRIPT
.find("elif video_negative_rows or audio_negative_rows:")
.expect("manual sync script should detect observed-before-injection rows");
let negative_rows_end = SYNC_SCRIPT[negative_rows_start..]
.find("span_gap_s = finite")
.map(|offset| negative_rows_start + offset)
.expect("negative-row invalidation should precede span diagnostics");
let negative_rows_block = &SYNC_SCRIPT[negative_rows_start..negative_rows_end];
for expected in [
"capture_timebase_status = \"invalid\"",
"video observed before server pipeline injection",
"audio observed before server pipeline injection",
"capture_timebase_reasons.append(",
] {
assert!(
negative_rows_block.contains(expected),
"real event timing contradictions must remain hard failures: {expected}"
);
}
for expected in [
"elif capture_timebase_status == \"invalid\":",
"freshness_reason = f\"capture timebase invalid: {capture_timebase_reason}\"",
"paired coded events {len(joined)} < {min_freshness_pairs}",
"freshness requires enough matched injected/observed events",
"server/capture clock alignment unavailable or too uncertain",
"non-monotonic event timestamps",
"minimum freshness",
"freshness was negative beyond clock uncertainty",
] {
assert!(
SYNC_SCRIPT.contains(expected),
"freshness should still fail on impossible event timing: {expected}"
);
}
}
#[test]
fn server_rc_mode_matrix_validates_advertised_uvc_profiles() {
for expected in [
"LESAVKA_SERVER_RC_CORE_WEBCAM_MODES=${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES:-1280x720@20,1280x720@30,1920x1080@20,1920x1080@30}",
"LESAVKA_SERVER_RC_MODES=${LESAVKA_SERVER_RC_MODES:-${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES}}",
"LESAVKA_SERVER_REPO=${LESAVKA_SERVER_REPO:-auto}",
"LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US:-${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}}",
"LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US:-170000}",
"LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US:-1280x720@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1280x720@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US}}",
"LESAVKA_SERVER_RC_MODE_DELAYS_US=${LESAVKA_SERVER_RC_MODE_DELAYS_US:-1280x720@20=170000,1280x720@30=170000,1920x1080@20=170000,1920x1080@30=170000}",
"LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES=${LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES:-1280x720,1920x1080}",
"LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS=${LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS:-20,30}",
"LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX:-Logitech|BRIO|C9[0-9]+|HD UVC WebCam|USB2[.]0 HD|Integrated Camera|Webcam|Camera}",
"LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX:-Lesavka|UGREEN|MACROSILICON|Composite|Capture}",
"LESAVKA_SERVER_RC_MODE_SOURCE=${LESAVKA_SERVER_RC_MODE_SOURCE:-configured}",
"LESAVKA_SERVER_RC_RECONFIGURE=${LESAVKA_SERVER_RC_RECONFIGURE:-0}",
"LESAVKA_SERVER_RC_RECONFIGURE_UPDATE=${LESAVKA_SERVER_RC_RECONFIGURE_UPDATE:-0}",
"LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY:-runtime}",
"LESAVKA_SERVER_RC_ALLOW_GADGET_RESET=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET:-1}",
"LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE=${LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE:-0}",
"LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY=${LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY:-1}",
"LESAVKA_SERVER_RC_WAIT_TETHYS_READY=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY:-1}",
"LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS:-60}",
"LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS:-6}",
"LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS:-3}",
"LESAVKA_SERVER_RC_PROBE_PREBUILD=${LESAVKA_SERVER_RC_PROBE_PREBUILD:-1}",
"LESAVKA_SERVER_RC_TUNE_DELAYS=${LESAVKA_SERVER_RC_TUNE_DELAYS:-1}",
"LESAVKA_SERVER_RC_TUNE_CONFIRM=${LESAVKA_SERVER_RC_TUNE_CONFIRM:-1}",
"LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-13}",
"LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS:-1000}",
"LESAVKA_SERVER_RC_TUNE_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US:-500000}",
"LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000}",
"Theia sudo password for %s",
"==> priming remote sudo on ${LESAVKA_SERVER_HOST}",
"==> prebuilding relay control/analyzer once for the mode matrix",
"LESAVKA_SERVER_RC_MODES=auto",
"discover_local_webcam_modes",
"lookup_audio_delay_us",
"local webcam",
"mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}",
"video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}",
"audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}",
"pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL}",
"fast runtime env updated: CAM_OUTPUT=uvc",
"cycling UVC gadget descriptors",
"lesavka-core reconfigure log:",
"missing /usr/local/bin/lesavka-core.sh",
"wait_tethys_media_ready",
"==> waiting for Tethys media endpoints for ${mode}",
"Tethys media ready: video=%s mode=%s audio_stack=%s",
"timed out waiting for Tethys Lesavka media endpoints",
"LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350}",
"LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS=${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS:-${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}}",
"LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS=${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS:-0}",
"LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS=${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS:-0}",
"LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES:-12}",
"mode-matrix-summary.json",
"mode-matrix-summary.csv",
"mode-matrix-summary.txt",
"mode-delay-recommendations.json",
"mode-delay-recommendations.env",
"schema\": \"lesavka.server-rc-mode-result.v1\"",
"schema\": \"lesavka.server-rc-mode-matrix-summary.v1\"",
"schema\": \"lesavka.server-rc-mode-delay-recommendations.v1\"",
"output_delay_calibration",
"write_tune_candidate_env",
"mode-result-seed.json",
"mode-result-tuned.json",
"==> mode ${mode}: confirming tuned delays",
"calibration_ready",
"calibration_video_target_offset_us",
"calibration_audio_target_offset_us",
"calibration:",
"capture_timebase_status",
"capture timebase invalid",
"capture timing:",
"signature_coverage",
"paired coded signatures",
"signature_missing_codes",
"REMOTE_PULSE_CAPTURE_TOOL=\"${REMOTE_PULSE_CAPTURE_TOOL}\"",
"REMOTE_PULSE_VIDEO_MODE=\"${REMOTE_PULSE_VIDEO_MODE}\"",
"REMOTE_CAPTURE_STACK=\"${REMOTE_CAPTURE_STACK}\"",
"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=\"${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}\"",
"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=\"${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}\"",
"REMOTE_CAPTURE_READY_SETTLE_SECONDS=\"${REMOTE_CAPTURE_READY_SETTLE_SECONDS}\"",
"PROBE_PREBUILD=0",
"VIDEO_SIZE=\"${width}x${height}\"",
"VIDEO_FPS=\"${fps}\"",
"REMOTE_EXPECT_UVC_WIDTH=\"${width}\"",
"REMOTE_EXPECT_UVC_HEIGHT=\"${height}\"",
"REMOTE_EXPECT_UVC_FPS=\"${fps}\"",
"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=\"${audio_delay_us}\"",
"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=\"${video_delay_us}\"",
"LESAVKA_OUTPUT_DELAY_APPLY=0",
"LESAVKA_OUTPUT_DELAY_SAVE=0",
"LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=\"${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}\"",
"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=\"${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}\"",
"LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=\"${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}\"",
"sync did not pass",
"freshness did not pass",
"video hiccups",
"estimated missing video frames",
"audio hiccups",
] {
assert!(
SERVER_RC_MODE_MATRIX_SCRIPT.contains(expected),
"server-to-RC mode matrix script should contain {expected}"
);
}
}
#[test]
fn browser_sync_script_can_delegate_to_a_real_path_driver() {
for expected in [
"BROWSER_RECORD_SECONDS=${BROWSER_RECORD_SECONDS:-${PROBE_DURATION_SECONDS}}",
"BROWSER_SYNC_DRIVER_COMMAND=${BROWSER_SYNC_DRIVER_COMMAND:-}",
"BROWSER_CONSUMER_REUSE_SESSION=${BROWSER_CONSUMER_REUSE_SESSION:-0}",
"BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED:-1}",
"SYNC_ANALYZE_EVENT_WIDTH_CODES=${SYNC_ANALYZE_EVENT_WIDTH_CODES:-}",
"==> running custom browser sync driver",
"bash -lc \"${BROWSER_SYNC_DRIVER_COMMAND}\"",
"browser_start_token=${browser_start_token}",
"uploaded_start_token",
"BROWSER_START_TOKEN",
"analysis-failure.json",
"BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED}",
"BROWSER_START_ATTEMPTS=${BROWSER_START_ATTEMPTS:-5}",
"browser consumer start attempt ${attempt}/${BROWSER_START_ATTEMPTS} failed; retrying",
"--event-width-codes",
"--report-dir \"${LOCAL_REPORT_DIR}\"",
"for attempt in 1 2 3 4 5",
"capture fetch attempt ${attempt} failed; retrying",
"failed to fetch browser capture from ${TETHYS_HOST}:${REMOTE_CAPTURE}",
r"raw activity delta was ([+-]?[0-9]+(?:\.[0-9]+)?) ms ",
r"\(video=([0-9]+(?:\.[0-9]+)?)s audio=([0-9]+(?:\.[0-9]+)?)s\)",
] {
assert!(
BROWSER_SYNC_SCRIPT.contains(expected),
"browser sync script should contain {expected}"
);
}
assert!(
!BROWSER_SYNC_SCRIPT.contains(r"(?:\\.[0-9]+)?"),
"browser sync raw-delta parser should not require a literal backslash before decimals"
);
}
#[test]
fn sync_probe_runner_uses_bundled_webcam_media_path() {
for expected in [
"bundled_webcam_media",
"refusing to measure split upstream",
"UpstreamMediaBundle",
"stream_webcam_media",
"PROBE_BUNDLE_SESSION_ID",
"PROBE_BUNDLE_AUDIO_GRACE",
"server does not advertise bundled webcam media",
] {
assert!(
SYNC_PROBE_RUNNER.contains(expected),
"sync probe runner should contain {expected}"
);
}
for forbidden in [
".stream_camera(Request::new(outbound))",
".stream_microphone(Request::new(outbound))",
] {
assert!(
!SYNC_PROBE_RUNNER.contains(forbidden),
"sync probe runner must not use old split upstream RPC {forbidden}"
);
}
}
#[test]
fn mirrored_sync_script_uses_real_client_capture_path() {
for expected in [
"local_av_stimulus.py",
"lesavka-client",
"LESAVKA_HEADLESS=1",
"LESAVKA_MEDIA_CONTROL=\"${MEDIA_CONTROL}\"",
"LESAVKA_REQUIRE_EXPLICIT_MEDIA_SOURCES=\"${LESAVKA_REQUIRE_EXPLICIT_MEDIA_SOURCES:-1}\"",
"--no-launcher --server \"${RESOLVED_LESAVKA_SERVER_ADDR}\"",
"BROWSER_SYNC_DRIVER_COMMAND=\"${driver_command}\"",
"SYNC_ANALYZE_EVENT_WIDTH_CODES=\"${PROBE_EVENT_WIDTH_CODES}\"",
"run_upstream_browser_av_sync.sh",
"wait_for_stimulus_page_ready 15",
"Point the real webcam at the stimulus window",
"==> Lesavka versions under test",
"lesavka-relayctl",
"--bin lesavka-relayctl",
"client_revision=",
"server_version=",
"server_revision=",
"combined version+revision",
"run_status=0",
"run_mirrored_segments || run_status=$?",
"LESAVKA_SYNC_APPLY_CALIBRATION=${LESAVKA_SYNC_APPLY_CALIBRATION:-0}",
"LESAVKA_SYNC_SAVE_CALIBRATION=${LESAVKA_SYNC_SAVE_CALIBRATION:-0}",
"LESAVKA_SYNC_CALIBRATION_TARGET=${LESAVKA_SYNC_CALIBRATION_TARGET:-video}",
"LESAVKA_SYNC_ADAPTIVE_CALIBRATION=${LESAVKA_SYNC_ADAPTIVE_CALIBRATION:-0}",
"LESAVKA_SYNC_CALIBRATION_SEGMENTS=${LESAVKA_SYNC_CALIBRATION_SEGMENTS:-1}",
"LESAVKA_SYNC_CONTINUOUS_BROWSER=${LESAVKA_SYNC_CONTINUOUS_BROWSER:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
"LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE=${LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
"LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS=${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS:-3}",
"PROBE_AUDIO_GAIN=${PROBE_AUDIO_GAIN:-0.55}",
"LESAVKA_STIMULUS_PREVIEW_SECONDS=${LESAVKA_STIMULUS_PREVIEW_SECONDS:-4}",
"LESAVKA_OPEN_MANUAL_REVIEW_DOLPHIN=${LESAVKA_OPEN_MANUAL_REVIEW_DOLPHIN:-1}",
"LESAVKA_SYNC_PROVISIONAL_CALIBRATION=${LESAVKA_SYNC_PROVISIONAL_CALIBRATION:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
"LESAVKA_SYNC_PROVISIONAL_MIN_PAIRS=${LESAVKA_SYNC_PROVISIONAL_MIN_PAIRS:-3}",
"LESAVKA_SYNC_PROVISIONAL_MAX_P95_MS=${LESAVKA_SYNC_PROVISIONAL_MAX_P95_MS:-350}",
"LESAVKA_SYNC_PROVISIONAL_MAX_DRIFT_MS=${LESAVKA_SYNC_PROVISIONAL_MAX_DRIFT_MS:-250}",
"LESAVKA_SYNC_PROVISIONAL_GAIN=${LESAVKA_SYNC_PROVISIONAL_GAIN:-0.5}",
"LESAVKA_SYNC_PROVISIONAL_MAX_STEP_US=${LESAVKA_SYNC_PROVISIONAL_MAX_STEP_US:-150000}",
"LESAVKA_SYNC_RAW_FAILURE_CALIBRATION=${LESAVKA_SYNC_RAW_FAILURE_CALIBRATION:-0}",
"LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS=${LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS:-3}",
"LESAVKA_SYNC_RAW_FAILURE_MAX_ABS_DELTA_MS=${LESAVKA_SYNC_RAW_FAILURE_MAX_ABS_DELTA_MS:-350}",
"LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION=${LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
"LESAVKA_SYNC_CONFIRMATION_SEGMENTS=${LESAVKA_SYNC_CONFIRMATION_SEGMENTS:-1}",
"LESAVKA_SYNC_REQUIRE_CONFIRMATION_PASS=${LESAVKA_SYNC_REQUIRE_CONFIRMATION_PASS:-${LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION}}",
"LESAVKA_SYNC_CONFIRMATION_SEGMENTS must be a non-negative integer",
"LESAVKA_SYNC_TOTAL_SEGMENTS=$((LESAVKA_SYNC_CALIBRATION_SEGMENTS + LESAVKA_SYNC_CONFIRMATION_SEGMENTS))",
"export LESAVKA_SYNC_PROVISIONAL_CALIBRATION",
"export LESAVKA_SYNC_RAW_FAILURE_CALIBRATION",
"export LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS",
"LESAVKA_SYNC_ADAPTIVE_CALIBRATION",
"LESAVKA_SYNC_CALIBRATION_SEGMENTS=4",
"browser_consumer_reuse_session=${reuse_browser_session}",
"browser_analysis_required=${analysis_required}",
"BROWSER_CONSUMER_REUSE_SESSION=\"${reuse_browser_session}\"",
"BROWSER_ANALYSIS_REQUIRED=\"${analysis_required}\"",
"--audio-gain \"${PROBE_AUDIO_GAIN}\"",
"run_stimulus_preview",
"write_stimulus_driver_script",
"local_stimulus_started=true",
"observed_start_token",
"audio_state",
"LESAVKA_SYNC_CALIBRATION_SEGMENTS must be a positive integer",
"run_mirrored_segments",
"summarize_adaptive_probe_metrics",
"for segment in $(seq 1 \"${LESAVKA_SYNC_TOTAL_SEGMENTS}\")",
"segment_phase",
"confirmation segment: calibration apply disabled so this segment tests the active calibration",
"segment-${segment}",
"calibration-before.env",
"planner-before.env",
"calibration-decision.env",
"segment-metrics.csv",
"segment-metrics.jsonl",
"segment-events.csv",
"segment-events.jsonl",
"manual-review",
"manual_review_html",
"manual_review_dir",
"open_manual_review_in_dolphin",
"dolphin",
"capture_path",
"confirmation-summary.json",
"confirmation_passed",
"check_confirmation_result",
"confirmation check failed",
"analysis_failure_reason",
"probe_activity_start_delta_ms",
"blind-targets.json",
"no segment produced a passing probe verdict; refusing to invent blind targets",
"confirmation did not pass; refusing to promote calibration-only segments to blind targets",
"candidate_good_calibration_segments",
"decision_mode",
"decision_provisional_video_recommendation_us",
"planner_live_lag_ms_after",
"probe_p95_abs_skew_ms",
"transport/server receive jitter",
"settling ${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS}s before next segment",
"print_upstream_calibration_state \"before mirrored run\"",
"maybe_apply_probe_calibration",
"calibration_ready=${calibration_ready}",
"calibration_decision_mode=${calibration_decision_mode}",
"bounded provisional correction from median skew",
"bounded provisional correction from analyzer-failure raw activity",
"raw_failure_calibration_enabled",
"raw analyzer-failure calibration refused: ",
"raw_failure_min_pairs",
"provisional calibration not saved",
"calibration apply refused: ${calibration_decision_note}",
"calibrate \"${calibration_apply_audio_delta_us}\" \"${calibration_apply_video_delta_us}\"",
"LESAVKA_UPSTREAM_BLIND_HEAL is server-side",
"calibration-save-default",
"print_upstream_sync_state \"after mirrored run\"",
"print_upstream_calibration_state \"after mirrored run\"",
"==> mirrored probe failed",
] {
assert!(
MIRRORED_SYNC_SCRIPT.contains(expected),
"mirrored sync script should contain {expected}"
);
}
assert!(
!MIRRORED_SYNC_SCRIPT.contains("lesavka-sync-probe"),
"mirrored sync must not use the synthetic direct sender"
);
}
#[test]
fn local_stimulus_matches_sync_analyzer_pulse_contract() {
for expected in [
"--warmup-seconds",
"--pulse-period-ms",
"--pulse-width-ms",
"--marker-tick-period",
"--audio-gain",
"/preview",
"preview_token",
"observed_preview_token",
"completed_preview_token",
"stimulus preview running",
"stimulus preview completed",
"audio_state",
"--event-width-codes",
"event_width_codes",
"audio_gain",
"widthCode",
"oscillator.frequency.value = 880",
"setStatus(`ready",
"Point the real webcam at this window",
] {
assert!(
LOCAL_STIMULUS.contains(expected),
"local stimulus should contain {expected}"
);
}
}
#[test]
fn manual_probe_python_servers_use_reentrant_state_locks() {
const BROWSER_CONSUMER: &str = include_str!("../../scripts/manual/browser_consumer_probe.py");
for (name, script) in [
("local stimulus", LOCAL_STIMULUS),
("browser consumer", BROWSER_CONSUMER),
] {
assert!(
script.contains("threading.RLock()"),
"{name} server request handlers call snapshot while holding state lock; use RLock to avoid /start deadlocks"
);
}
}