//! Contract tests for the manual upstream A/V sync harness. //! //! Scope: statically guard the workstation-side tunnel/bootstrap behavior. //! Targets: `scripts/manual/run_upstream_av_sync.sh`. //! Why: the manual probe should reach Theia through SSH even when the gRPC //! port is not exposed on the public SSH endpoint. const SYNC_SCRIPT: &str = include_str!("../../scripts/manual/run_upstream_av_sync.sh"); const SERVER_RC_MODE_MATRIX_SCRIPT: &str = include_str!("../../scripts/manual/run_server_to_rc_mode_matrix.sh"); const BROWSER_SYNC_SCRIPT: &str = include_str!("../../scripts/manual/run_upstream_browser_av_sync.sh"); const MIRRORED_SYNC_SCRIPT: &str = include_str!("../../scripts/manual/run_upstream_mirrored_av_sync.sh"); const LOCAL_STIMULUS: &str = include_str!("../../scripts/manual/local_av_stimulus.py"); const SYNC_PROBE_RUNNER: &str = include_str!("../../client/src/sync_probe/runner.rs"); #[test] fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() { for expected in [ "cleanup_server_tunnel", "pick_local_server_tunnel_port", "wait_for_server_tunnel", "start_server_tunnel", "ExitOnForwardFailure=yes", "127.0.0.1:${local_port}:127.0.0.1:${remote_port}", "LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https}", "LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}", "RESOLVED_LESAVKA_SERVER_ADDR=\"${LESAVKA_SERVER_SCHEME}://127.0.0.1:${SERVER_TUNNEL_LOCAL_PORT}\"", "LESAVKA_TLS_DOMAIN=\"${LESAVKA_TLS_DOMAIN}\"", "tunneled to ${LESAVKA_SERVER_HOST}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT}", "CAPTURE_READY_MARKER=\"__LESAVKA_CAPTURE_READY__\"", "LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}", "LOCAL_REPORT_DIR=\"${LOCAL_OUTPUT_DIR%/}/lesavka-output-delay-probe-${STAMP}\"", "LOCAL_ANALYSIS_JSON=\"${LOCAL_REPORT_DIR}/report.json\"", "LOCAL_ANALYSIS_ERROR_LOG=\"${LOCAL_REPORT_DIR}/analysis-error.log\"", "LOCAL_EVENTS_CSV=\"${LOCAL_REPORT_DIR}/events.csv\"", "LOCAL_SERVER_PROBE_REPLY=\"${LOCAL_REPORT_DIR}/server-output-probe-reply.txt\"", "LOCAL_SERVER_TIMELINE_JSON=\"${LOCAL_REPORT_DIR}/server-output-timeline.json\"", "LOCAL_SIGNAL_CONDITIONING_REPLY=\"${LOCAL_REPORT_DIR}/signal-conditioning-probe-reply.txt\"", "LOCAL_SIGNAL_CONDITIONING_TIMELINE_JSON=\"${LOCAL_REPORT_DIR}/signal-conditioning-timeline.json\"", "LOCAL_OUTPUT_DELAY_CORRELATION_JSON=\"${LOCAL_REPORT_DIR}/output-delay-correlation.json\"", "LOCAL_OUTPUT_DELAY_CORRELATION_CSV=\"${LOCAL_REPORT_DIR}/output-delay-correlation.csv\"", "LOCAL_OUTPUT_DELAY_CORRELATION_TXT=\"${LOCAL_REPORT_DIR}/output-delay-correlation.txt\"", "LOCAL_CLOCK_ALIGNMENT_JSON=\"${LOCAL_REPORT_DIR}/clock-alignment.json\"", "LOCAL_OUTPUT_DELAY_JSON=\"${LOCAL_REPORT_DIR}/output-delay-calibration.json\"", "LOCAL_OUTPUT_DELAY_ENV=\"${LOCAL_REPORT_DIR}/output-delay-calibration.env\"", "LESAVKA_OUTPUT_DELAY_CALIBRATION=${LESAVKA_OUTPUT_DELAY_CALIBRATION:-1}", "LESAVKA_OUTPUT_DELAY_APPLY=${LESAVKA_OUTPUT_DELAY_APPLY:-0}", "LESAVKA_OUTPUT_DELAY_APPLY_MODE=${LESAVKA_OUTPUT_DELAY_APPLY_MODE:-absolute}", "LESAVKA_OUTPUT_DELAY_CONFIRM=${LESAVKA_OUTPUT_DELAY_CONFIRM:-1}", "LESAVKA_OUTPUT_DELAY_SAVE=${LESAVKA_OUTPUT_DELAY_SAVE:-0}", "LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=${LESAVKA_OUTPUT_REQUIRE_SYNC_PASS:-0}", "LESAVKA_OUTPUT_DELAY_TARGET=${LESAVKA_OUTPUT_DELAY_TARGET:-video}", "LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${LESAVKA_OUTPUT_DELAY_MIN_PAIRS:-13}", "LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS:-5000}", "LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS=${LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS:-80}", "LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_OUTPUT_DELAY_MAX_STEP_US:-1500000}", "LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS:-1000}", "LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS:-100}", "LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}", "LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=${LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS:-${LESAVKA_OUTPUT_DELAY_MIN_PAIRS}}", "REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS:-0}", "REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS:-1}", "remote_capture_ready_settle_seconds=${15}", "announce_capture_start", "signal_capture_ready", "run_tolerant_capture", "discarding %ss of post-enumeration capture before probe", "ffmpeg -nostdin -hide_banner", "\"$@\" UVC/UAC sinks -> lab host capture\"", "audio_after_video_positive", "active_audio_offset_us", "active_video_offset_us", "audio_target_offset_us = active_audio_offset_us + audio_delta_us", "video_target_offset_us = active_video_offset_us + video_delta_us", "output_delay_active_audio_offset_us", "output_delay_active_video_offset_us", "audio_target_offset_us", "video_target_offset_us", "output_delay_audio_target_offset_us", "output_delay_video_target_offset_us", "LESAVKA_OUTPUT_DELAY_CONFIRMING=1", "LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=1", "LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=\"${confirm_audio_delay}\"", "LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=\"${confirm_video_delay}\"", "==> confirming fixed UVC/UAC output-delay calibration", "required sync pass failed", "calibration_active_video_offset_us", "absolute_target_video_offset_us", "calibration_apply_video_delta_us", "PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}", "\"${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US}\"", "\"${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}\"", "REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}", "REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}", "REMOTE_ARTIFACT_RETRIES=${REMOTE_ARTIFACT_RETRIES:-3}", "REMOTE_ARTIFACT_RETRY_DELAY_SECONDS=${REMOTE_ARTIFACT_RETRY_DELAY_SECONDS:-5}", "retry_remote_artifact_command", "run_remote_sync_analysis_once", "checking remote capture on ${TETHYS_HOST}", "copying sync analyzer to ${TETHYS_HOST}", "running remote sync analysis on ${TETHYS_HOST}", "fetching capture from ${TETHYS_HOST}", "warning: failed to fetch capture artifact; continuing with remote analysis JSON when available", "analysis_tmp=\"${LOCAL_ANALYSIS_JSON}.tmp\"", "analysis_error_tmp=\"${LOCAL_ANALYSIS_ERROR_LOG}.tmp\"", "analysis_error_log: ${LOCAL_ANALYSIS_ERROR_LOG}", "event_visibility", "Coded event visibility", "video_only", "audio_only", "REMOTE_PULSE_AUDIO_ANCHOR_SILENCE=${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE:-1}", "anchoring Pulse capture audio timeline with generated silence", "audiotestsrc wave=silence is-live=true do-timestamp=true", "audiomixer name=amix ignore-inactive-pads=true", "output_delay_calibration_json", "direct UVC/UAC output-delay calibration", "calibration-save-default", "LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-0}", "PROBE_START_GRACE_SECONDS=${PROBE_START_GRACE_SECONDS:-20}", "PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + PROBE_START_GRACE_SECONDS))}", "CAPTURE_START_GRACE_SECONDS=${CAPTURE_START_GRACE_SECONDS:-12}", "CONDITIONING_CAPTURE_SECONDS=$((PROBE_CONDITIONING_SECONDS > 0 ? PROBE_CONDITIONING_SECONDS + PROBE_CONDITIONING_WARMUP_SECONDS + PROBE_CONDITIONING_GAP_SECONDS : 0))", "CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + CONDITIONING_CAPTURE_SECONDS + CAPTURE_START_GRACE_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))}", "ANALYSIS_TIMELINE_WINDOW=${ANALYSIS_TIMELINE_WINDOW:-0}", "compute_analysis_window_arg", "analyzer timeline window:", "run_output_delay_probe", "conditioning Tethys UVC/UAC signal path before measured probe", "signal_conditioning_reply: ${LOCAL_SIGNAL_CONDITIONING_REPLY}", "signal_conditioning_timeline_json: ${LOCAL_SIGNAL_CONDITIONING_TIMELINE_JSON}", "output-delay-probe", "server-generated UVC/UAC output-delay probe", "server output-delay probe timed out after ${PROBE_TIMEOUT_SECONDS}s", "--event-width-codes '${PROBE_EVENT_WIDTH_CODES}'", "VIDIOC_STREAMON.*Connection timed out", "the UVC host opened before MJPEG frames reached the gadget", "Tethys capture failed before the sync probe could start", "wait_for_capture_ready", "Timed out waiting for Tethys capture to become ready", "grep -q \"${CAPTURE_READY_MARKER}\"", "Lesavka UVC video device not found on Tethys; refusing to fall back to an unrelated webcam/capture card.", "resolve_alsa_audio_device", "PipeWire Lesavka source not found; using explicit diagnostic ALSA fallback device", "Set REMOTE_CAPTURE_STACK=alsa or REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=1 only for diagnostic signal-presence checks.", "Lesavka Pulse/PipeWire audio source not found; refusing raw ALSA fallback for timing-sensitive capture.", "discarding %ss of post-enumeration capture before probe", "using Pulse source:", "-f pulse", "-map 1:a:0", "artifact_dir: ${LOCAL_REPORT_DIR}", "events_csv: ${LOCAL_EVENTS_CSV}", "server_timeline_json: ${LOCAL_SERVER_TIMELINE_JSON}", "output_delay_correlation_json: ${LOCAL_OUTPUT_DELAY_CORRELATION_JSON}", "==> Lesavka versions under test", "lesavka-relayctl", "--bin lesavka-relayctl", "--bin lesavka-sync-analyze", "client_revision=", "server_version=", "server_revision=", "combined version+revision", ] { assert!( SYNC_SCRIPT.contains(expected), "manual sync script should contain {expected}" ); } assert!( !SYNC_SCRIPT.contains( "RESOLVED_LESAVKA_SERVER_ADDR=\"http://${LESAVKA_SERVER_CONNECT_HOST}:${port}\"" ), "auto server resolution should not guess a public gRPC host when SSH is already required" ); for forbidden in [ "LOCAL_AUDIO_SANITY", "run_local_audio_sanity.sh", "lesavka-sync-probe", ] { assert!( !SYNC_SCRIPT.contains(forbidden), "direct UVC/UAC output-delay probe must not run workstation/client-side probe behavior: {forbidden}" ); } } #[test] fn output_freshness_is_event_level_not_capture_container_duration() { for expected in [ "\"freshness_clock_starts_at\": \"server pipeline reference before intentional sync delay\"", "video_freshness_ms = (", "audio_freshness_ms = (", "\"video_freshness_stats\": video_freshness_stats", "\"audio_freshness_stats\": audio_freshness_stats", "\"worst_p95_pipeline_freshness_ms\": freshness_worst_p95_ms", "\"worst_event_age_with_uncertainty_ms\": freshness_worst_event_with_uncertainty_ms", "\"configured_capture_duration_s\": configured_capture_duration_s", "\"observed_capture_duration_s\": observed_capture_duration_s", "\"min_paired_events\": min_freshness_pairs", "\"paired_event_count\": len(joined)", "\"warnings\": capture_timebase_warnings", "treating whole-file media span as recorder coverage, not event freshness", "Freshness timeline", "measured event freshness", ] { assert!( SYNC_SCRIPT.contains(expected), "freshness contract should contain event-level timing marker {expected}" ); } let span_gap_start = SYNC_SCRIPT .find("span_gap_s = finite(media_timeline.get(\"audio_video_span_gap_s\"))") .expect("manual sync script should compute media span gap diagnostics"); let span_gap_end = SYNC_SCRIPT[span_gap_start..] .find("if capture_timebase_reasons:") .map(|offset| span_gap_start + offset) .expect("span gap diagnostic block should end before reason folding"); let span_gap_block = &SYNC_SCRIPT[span_gap_start..span_gap_end]; assert!( span_gap_block.contains("capture_timebase_warnings.append("), "whole-file media span mismatch should be recorded as a warning" ); assert!( !span_gap_block.contains("capture_timebase_status = \"invalid\""), "whole-file media span mismatch must not invalidate freshness; freshness is event-level" ); assert!( !span_gap_block.contains("capture_timebase_reasons.append("), "whole-file media span mismatch must not become a hard invalidation reason" ); } #[test] fn output_freshness_still_invalidates_real_event_timing_contradictions() { let negative_rows_start = SYNC_SCRIPT .find("elif video_negative_rows or audio_negative_rows:") .expect("manual sync script should detect observed-before-injection rows"); let negative_rows_end = SYNC_SCRIPT[negative_rows_start..] .find("span_gap_s = finite") .map(|offset| negative_rows_start + offset) .expect("negative-row invalidation should precede span diagnostics"); let negative_rows_block = &SYNC_SCRIPT[negative_rows_start..negative_rows_end]; for expected in [ "capture_timebase_status = \"invalid\"", "video observed before server pipeline injection", "audio observed before server pipeline injection", "capture_timebase_reasons.append(", ] { assert!( negative_rows_block.contains(expected), "real event timing contradictions must remain hard failures: {expected}" ); } for expected in [ "elif capture_timebase_status == \"invalid\":", "freshness_reason = f\"capture timebase invalid: {capture_timebase_reason}\"", "paired coded events {len(joined)} < {min_freshness_pairs}", "freshness requires enough matched injected/observed events", "server/capture clock alignment unavailable or too uncertain", "non-monotonic event timestamps", "minimum freshness", "freshness was negative beyond clock uncertainty", ] { assert!( SYNC_SCRIPT.contains(expected), "freshness should still fail on impossible event timing: {expected}" ); } } #[test] fn server_rc_mode_matrix_validates_advertised_uvc_profiles() { for expected in [ "LESAVKA_SERVER_RC_CORE_WEBCAM_MODES=${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES:-1280x720@20,1280x720@30,1920x1080@20,1920x1080@30}", "LESAVKA_SERVER_RC_MODES=${LESAVKA_SERVER_RC_MODES:-${LESAVKA_SERVER_RC_CORE_WEBCAM_MODES}}", "LESAVKA_SERVER_REPO=${LESAVKA_SERVER_REPO:-auto}", "LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US:-${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}}", "LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US=${LESAVKA_SERVER_RC_DEFAULT_VIDEO_DELAY_US:-170000}", "LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US:-1280x720@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1280x720@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@20=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US},1920x1080@30=${LESAVKA_SERVER_RC_DEFAULT_AUDIO_DELAY_US}}", "LESAVKA_SERVER_RC_MODE_DELAYS_US=${LESAVKA_SERVER_RC_MODE_DELAYS_US:-1280x720@20=170000,1280x720@30=170000,1920x1080@20=170000,1920x1080@30=170000}", "LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES=${LESAVKA_SERVER_RC_MODE_DISCOVERY_SIZES:-1280x720,1920x1080}", "LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS=${LESAVKA_SERVER_RC_MODE_DISCOVERY_FPS:-20,30}", "LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_INCLUDE_REGEX:-Logitech|BRIO|C9[0-9]+|HD UVC WebCam|USB2[.]0 HD|Integrated Camera|Webcam|Camera}", "LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX=${LESAVKA_SERVER_RC_MODE_DISCOVERY_EXCLUDE_REGEX:-Lesavka|UGREEN|MACROSILICON|Composite|Capture}", "LESAVKA_SERVER_RC_MODE_SOURCE=${LESAVKA_SERVER_RC_MODE_SOURCE:-configured}", "LESAVKA_SERVER_RC_RECONFIGURE=${LESAVKA_SERVER_RC_RECONFIGURE:-0}", "LESAVKA_SERVER_RC_RECONFIGURE_UPDATE=${LESAVKA_SERVER_RC_RECONFIGURE_UPDATE:-0}", "LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY:-runtime}", "LESAVKA_SERVER_RC_ALLOW_GADGET_RESET=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET:-1}", "LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE=${LESAVKA_SERVER_RC_RECONFIGURE_VERBOSE:-0}", "LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY=${LESAVKA_SERVER_RC_PROMPT_SUDO_EARLY:-1}", "LESAVKA_SERVER_RC_START_DELAY_SECONDS=${LESAVKA_SERVER_RC_START_DELAY_SECONDS:-0}", "LESAVKA_SERVER_RC_WAIT_TETHYS_READY=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY:-1}", "LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS:-60}", "LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS:-6}", "LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS:-3}", "LESAVKA_SERVER_RC_PROBE_PREBUILD=${LESAVKA_SERVER_RC_PROBE_PREBUILD:-1}", "LESAVKA_SERVER_RC_TUNE_DELAYS=${LESAVKA_SERVER_RC_TUNE_DELAYS:-1}", "LESAVKA_SERVER_RC_TUNE_CONFIRM=${LESAVKA_SERVER_RC_TUNE_CONFIRM:-1}", "LESAVKA_SERVER_RC_TUNE_MIN_PAIRS=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS:-13}", "LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS:-1000}", "LESAVKA_SERVER_RC_TUNE_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US:-500000}", "LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000}", "Theia sudo password for %s", "==> priming remote sudo on ${LESAVKA_SERVER_HOST}", "sleep_start_delay", "==> delaying server-to-RC matrix start for ${LESAVKA_SERVER_RC_START_DELAY_SECONDS}s", "remote sudo has already been primed; sleeping before prebuild/reconfigure/capture", "LESAVKA_SERVER_RC_START_DELAY_SECONDS must be a non-negative number", "start_delay=${LESAVKA_SERVER_RC_START_DELAY_SECONDS}s", "==> prebuilding relay control/analyzer once for the mode matrix", "LESAVKA_SERVER_RC_MODES=auto", "discover_local_webcam_modes", "lookup_audio_delay_us", "local webcam", "mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}", "video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}", "audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}", "pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL}", "fast runtime env updated: CAM_OUTPUT=uvc", "cycling UVC gadget descriptors", "lesavka-core reconfigure log:", "missing /usr/local/bin/lesavka-core.sh", "wait_tethys_media_ready", "==> waiting for Tethys media endpoints for ${mode}", "Tethys media ready: video=%s mode=%s audio_stack=%s", "timed out waiting for Tethys Lesavka media endpoints", "LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350}", "LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS=${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS:-${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS}}", "LESAVKA_SERVER_RC_MIN_CODED_PAIRS=${LESAVKA_SERVER_RC_MIN_CODED_PAIRS:-${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}}", "LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS=${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS:-0}", "LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS:-0}", "LESAVKA_SERVER_RC_SIGNAL_READY=${LESAVKA_SERVER_RC_SIGNAL_READY:-1}", "LESAVKA_SERVER_RC_SIGNAL_READY_MODE=${LESAVKA_SERVER_RC_SIGNAL_READY_MODE:-conditioned_capture}", "LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS:-3}", "LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS:-12}", "LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS:-4}", "LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS:-5}", "LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_SECONDS:-12}", "LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS:-1}", "LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS:-1}", "LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW=${LESAVKA_SERVER_RC_ANALYSIS_TIMELINE_WINDOW:-auto}", "signal_readiness_passed", "write_signal_readiness_attempt_result", "write_signal_readiness_attempts_summary", "schema\": \"lesavka.server-rc-signal-readiness-attempt.v1\"", "schema\": \"lesavka.server-rc-signal-readiness-summary.v1\"", "using same-capture signal conditioning before measured probe", "proving Tethys signal readiness before measured probe", "readiness attempt ${readiness_attempt}/${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}", "waiting ${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s before retrying signal readiness", "signal readiness did not pass", "signal attempt ", "smoothness_required", "smoothness_warnings", "smoothness warning:", "coded visibility:", "LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS=${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS:-0}", "LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS=${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS:-0}", "LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES=${LESAVKA_SERVER_RC_MAX_VIDEO_MISSING_FRAMES:-12}", "mode-matrix-summary.json", "mode-matrix-summary.csv", "mode-matrix-summary.txt", "mode-delay-recommendations.json", "mode-delay-recommendations.env", "schema\": \"lesavka.server-rc-mode-result.v1\"", "schema\": \"lesavka.server-rc-mode-matrix-summary.v1\"", "schema\": \"lesavka.server-rc-mode-delay-recommendations.v1\"", "output_delay_calibration", "write_tune_candidate_env", "annotate_mode_result", "LESAVKA_SERVER_RC_REPEAT_COUNT=${LESAVKA_SERVER_RC_REPEAT_COUNT:-1}", "mode-static-calibration.json", "mode-matrix-run.log", "mode-result-seed.json", "mode-result-tuned.json", "==> mode ${mode} run ${mode_run_index}: confirming tuned delays", "calibration_ready", "calibration_video_target_offset_us", "calibration_audio_target_offset_us", "calibration:", "capture_timebase_status", "capture timebase invalid", "capture timing:", "signature_coverage", "paired coded signatures", "signature_missing_codes", "probe_env=(", "\"REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL}\"", "\"REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE}\"", "\"REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK}\"", "\"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}\"", "\"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}\"", "\"REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS}\"", "\"PROBE_PREBUILD=0\"", "\"VIDEO_SIZE=${width}x${height}\"", "\"VIDEO_FPS=${fps}\"", "\"REMOTE_EXPECT_UVC_WIDTH=${width}\"", "\"REMOTE_EXPECT_UVC_HEIGHT=${height}\"", "\"REMOTE_EXPECT_UVC_FPS=${fps}\"", "\"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${audio_delay_us}\"", "\"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${video_delay_us}\"", "\"LESAVKA_OUTPUT_DELAY_APPLY=0\"", "\"LESAVKA_OUTPUT_DELAY_SAVE=0\"", "\"LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}\"", "\"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}\"", "\"LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=${min_pairs}\"", "sync did not pass", "freshness did not pass", "video hiccups", "estimated missing video frames", "audio hiccups", ] { assert!( SERVER_RC_MODE_MATRIX_SCRIPT.contains(expected), "server-to-RC mode matrix script should contain {expected}" ); } let prime = SERVER_RC_MODE_MATRIX_SCRIPT .find("prime_remote_sudo\nsleep_start_delay\nprebuild_probe_tools") .expect("matrix should prime remote sudo before delayed start and prebuild"); let prompt = SERVER_RC_MODE_MATRIX_SCRIPT .find("Theia sudo password for %s") .expect("matrix should contain the immediate Theia password prompt"); assert!( prompt < prime, "password prompt machinery should be defined before the matrix startup sequence" ); } #[test] fn browser_sync_script_can_delegate_to_a_real_path_driver() { for expected in [ "BROWSER_RECORD_SECONDS=${BROWSER_RECORD_SECONDS:-${PROBE_DURATION_SECONDS}}", "BROWSER_SYNC_DRIVER_COMMAND=${BROWSER_SYNC_DRIVER_COMMAND:-}", "BROWSER_CONSUMER_REUSE_SESSION=${BROWSER_CONSUMER_REUSE_SESSION:-0}", "BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED:-1}", "SYNC_ANALYZE_EVENT_WIDTH_CODES=${SYNC_ANALYZE_EVENT_WIDTH_CODES:-}", "==> running custom browser sync driver", "bash -lc \"${BROWSER_SYNC_DRIVER_COMMAND}\"", "browser_start_token=${browser_start_token}", "uploaded_start_token", "BROWSER_START_TOKEN", "analysis-failure.json", "BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED}", "BROWSER_START_ATTEMPTS=${BROWSER_START_ATTEMPTS:-5}", "browser consumer start attempt ${attempt}/${BROWSER_START_ATTEMPTS} failed; retrying", "--event-width-codes", "--report-dir \"${LOCAL_REPORT_DIR}\"", "for attempt in 1 2 3 4 5", "capture fetch attempt ${attempt} failed; retrying", "failed to fetch browser capture from ${TETHYS_HOST}:${REMOTE_CAPTURE}", r"raw activity delta was ([+-]?[0-9]+(?:\.[0-9]+)?) ms ", r"\(video=([0-9]+(?:\.[0-9]+)?)s audio=([0-9]+(?:\.[0-9]+)?)s\)", ] { assert!( BROWSER_SYNC_SCRIPT.contains(expected), "browser sync script should contain {expected}" ); } assert!( !BROWSER_SYNC_SCRIPT.contains(r"(?:\\.[0-9]+)?"), "browser sync raw-delta parser should not require a literal backslash before decimals" ); } #[test] fn sync_probe_runner_uses_bundled_webcam_media_path() { for expected in [ "bundled_webcam_media", "refusing to measure split upstream", "UpstreamMediaBundle", "stream_webcam_media", "PROBE_BUNDLE_SESSION_ID", "PROBE_BUNDLE_AUDIO_GRACE", "server does not advertise bundled webcam media", ] { assert!( SYNC_PROBE_RUNNER.contains(expected), "sync probe runner should contain {expected}" ); } for forbidden in [ ".stream_camera(Request::new(outbound))", ".stream_microphone(Request::new(outbound))", ] { assert!( !SYNC_PROBE_RUNNER.contains(forbidden), "sync probe runner must not use old split upstream RPC {forbidden}" ); } } #[test] fn mirrored_sync_script_uses_real_client_capture_path() { for expected in [ "local_av_stimulus.py", "lesavka-client", "LESAVKA_HEADLESS=1", "LESAVKA_MEDIA_CONTROL=\"${MEDIA_CONTROL}\"", "LESAVKA_REQUIRE_EXPLICIT_MEDIA_SOURCES=\"${LESAVKA_REQUIRE_EXPLICIT_MEDIA_SOURCES:-1}\"", "--no-launcher --server \"${RESOLVED_LESAVKA_SERVER_ADDR}\"", "BROWSER_SYNC_DRIVER_COMMAND=\"${driver_command}\"", "SYNC_ANALYZE_EVENT_WIDTH_CODES=\"${PROBE_EVENT_WIDTH_CODES}\"", "run_upstream_browser_av_sync.sh", "wait_for_stimulus_page_ready 15", "Point the real webcam at the stimulus window", "==> Lesavka versions under test", "lesavka-relayctl", "--bin lesavka-relayctl", "client_revision=", "server_version=", "server_revision=", "combined version+revision", "run_status=0", "run_mirrored_segments || run_status=$?", "LESAVKA_SYNC_APPLY_CALIBRATION=${LESAVKA_SYNC_APPLY_CALIBRATION:-0}", "LESAVKA_SYNC_SAVE_CALIBRATION=${LESAVKA_SYNC_SAVE_CALIBRATION:-0}", "LESAVKA_SYNC_CALIBRATION_TARGET=${LESAVKA_SYNC_CALIBRATION_TARGET:-video}", "LESAVKA_SYNC_ADAPTIVE_CALIBRATION=${LESAVKA_SYNC_ADAPTIVE_CALIBRATION:-0}", "LESAVKA_SYNC_CALIBRATION_SEGMENTS=${LESAVKA_SYNC_CALIBRATION_SEGMENTS:-1}", "LESAVKA_SYNC_CONTINUOUS_BROWSER=${LESAVKA_SYNC_CONTINUOUS_BROWSER:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}", "LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE=${LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}", "LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS=${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS:-3}", "PROBE_AUDIO_GAIN=${PROBE_AUDIO_GAIN:-0.55}", "LESAVKA_STIMULUS_PREVIEW_SECONDS=${LESAVKA_STIMULUS_PREVIEW_SECONDS:-4}", "LESAVKA_OPEN_MANUAL_REVIEW_DOLPHIN=${LESAVKA_OPEN_MANUAL_REVIEW_DOLPHIN:-1}", "LESAVKA_SYNC_PROVISIONAL_CALIBRATION=${LESAVKA_SYNC_PROVISIONAL_CALIBRATION:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}", "LESAVKA_SYNC_PROVISIONAL_MIN_PAIRS=${LESAVKA_SYNC_PROVISIONAL_MIN_PAIRS:-3}", "LESAVKA_SYNC_PROVISIONAL_MAX_P95_MS=${LESAVKA_SYNC_PROVISIONAL_MAX_P95_MS:-350}", "LESAVKA_SYNC_PROVISIONAL_MAX_DRIFT_MS=${LESAVKA_SYNC_PROVISIONAL_MAX_DRIFT_MS:-250}", "LESAVKA_SYNC_PROVISIONAL_GAIN=${LESAVKA_SYNC_PROVISIONAL_GAIN:-0.5}", "LESAVKA_SYNC_PROVISIONAL_MAX_STEP_US=${LESAVKA_SYNC_PROVISIONAL_MAX_STEP_US:-150000}", "LESAVKA_SYNC_RAW_FAILURE_CALIBRATION=${LESAVKA_SYNC_RAW_FAILURE_CALIBRATION:-0}", "LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS=${LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS:-3}", "LESAVKA_SYNC_RAW_FAILURE_MAX_ABS_DELTA_MS=${LESAVKA_SYNC_RAW_FAILURE_MAX_ABS_DELTA_MS:-350}", "LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION=${LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}", "LESAVKA_SYNC_CONFIRMATION_SEGMENTS=${LESAVKA_SYNC_CONFIRMATION_SEGMENTS:-1}", "LESAVKA_SYNC_REQUIRE_CONFIRMATION_PASS=${LESAVKA_SYNC_REQUIRE_CONFIRMATION_PASS:-${LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION}}", "LESAVKA_SYNC_CONFIRMATION_SEGMENTS must be a non-negative integer", "LESAVKA_SYNC_TOTAL_SEGMENTS=$((LESAVKA_SYNC_CALIBRATION_SEGMENTS + LESAVKA_SYNC_CONFIRMATION_SEGMENTS))", "export LESAVKA_SYNC_PROVISIONAL_CALIBRATION", "export LESAVKA_SYNC_RAW_FAILURE_CALIBRATION", "export LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS", "LESAVKA_SYNC_ADAPTIVE_CALIBRATION", "LESAVKA_SYNC_CALIBRATION_SEGMENTS=4", "browser_consumer_reuse_session=${reuse_browser_session}", "browser_analysis_required=${analysis_required}", "BROWSER_CONSUMER_REUSE_SESSION=\"${reuse_browser_session}\"", "BROWSER_ANALYSIS_REQUIRED=\"${analysis_required}\"", "--audio-gain \"${PROBE_AUDIO_GAIN}\"", "run_stimulus_preview", "write_stimulus_driver_script", "local_stimulus_started=true", "observed_start_token", "audio_state", "LESAVKA_SYNC_CALIBRATION_SEGMENTS must be a positive integer", "run_mirrored_segments", "summarize_adaptive_probe_metrics", "for segment in $(seq 1 \"${LESAVKA_SYNC_TOTAL_SEGMENTS}\")", "segment_phase", "confirmation segment: calibration apply disabled so this segment tests the active calibration", "segment-${segment}", "calibration-before.env", "planner-before.env", "calibration-decision.env", "segment-metrics.csv", "segment-metrics.jsonl", "segment-events.csv", "segment-events.jsonl", "manual-review", "manual_review_html", "manual_review_dir", "open_manual_review_in_dolphin", "dolphin", "capture_path", "confirmation-summary.json", "confirmation_passed", "check_confirmation_result", "confirmation check failed", "analysis_failure_reason", "probe_activity_start_delta_ms", "blind-targets.json", "no segment produced a passing probe verdict; refusing to invent blind targets", "confirmation did not pass; refusing to promote calibration-only segments to blind targets", "candidate_good_calibration_segments", "decision_mode", "decision_provisional_video_recommendation_us", "planner_live_lag_ms_after", "probe_p95_abs_skew_ms", "transport/server receive jitter", "settling ${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS}s before next segment", "print_upstream_calibration_state \"before mirrored run\"", "maybe_apply_probe_calibration", "calibration_ready=${calibration_ready}", "calibration_decision_mode=${calibration_decision_mode}", "bounded provisional correction from median skew", "bounded provisional correction from analyzer-failure raw activity", "raw_failure_calibration_enabled", "raw analyzer-failure calibration refused: ", "raw_failure_min_pairs", "provisional calibration not saved", "calibration apply refused: ${calibration_decision_note}", "calibrate \"${calibration_apply_audio_delta_us}\" \"${calibration_apply_video_delta_us}\"", "LESAVKA_UPSTREAM_BLIND_HEAL is server-side", "calibration-save-default", "print_upstream_sync_state \"after mirrored run\"", "print_upstream_calibration_state \"after mirrored run\"", "==> mirrored probe failed", ] { assert!( MIRRORED_SYNC_SCRIPT.contains(expected), "mirrored sync script should contain {expected}" ); } assert!( !MIRRORED_SYNC_SCRIPT.contains("lesavka-sync-probe"), "mirrored sync must not use the synthetic direct sender" ); } #[test] fn local_stimulus_matches_sync_analyzer_pulse_contract() { for expected in [ "--warmup-seconds", "--pulse-period-ms", "--pulse-width-ms", "--marker-tick-period", "--audio-gain", "/preview", "preview_token", "observed_preview_token", "completed_preview_token", "stimulus preview running", "stimulus preview completed", "audio_state", "--event-width-codes", "event_width_codes", "audio_gain", "widthCode", "oscillator.frequency.value = 880", "setStatus(`ready", "Point the real webcam at this window", ] { assert!( LOCAL_STIMULUS.contains(expected), "local stimulus should contain {expected}" ); } } #[test] fn manual_probe_python_servers_use_reentrant_state_locks() { const BROWSER_CONSUMER: &str = include_str!("../../scripts/manual/browser_consumer_probe.py"); for (name, script) in [ ("local stimulus", LOCAL_STIMULUS), ("browser consumer", BROWSER_CONSUMER), ] { assert!( script.contains("threading.RLock()"), "{name} server request handlers call snapshot while holding state lock; use RLock to avoid /start deadlocks" ); } }