//! Contract tests for the manual upstream A/V sync harness. //! //! Scope: statically guard the workstation-side tunnel/bootstrap behavior. //! Targets: `scripts/manual/run_upstream_av_sync.sh`. //! Why: the manual probe should reach Theia through SSH even when the gRPC //! port is not exposed on the public SSH endpoint. const SYNC_SCRIPT: &str = include_str!("../../scripts/manual/run_upstream_av_sync.sh"); #[test] fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() { for expected in [ "cleanup_server_tunnel", "pick_local_server_tunnel_port", "wait_for_server_tunnel", "start_server_tunnel", "ExitOnForwardFailure=yes", "127.0.0.1:${local_port}:127.0.0.1:${remote_port}", "LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https}", "LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}", "RESOLVED_LESAVKA_SERVER_ADDR=\"${LESAVKA_SERVER_SCHEME}://127.0.0.1:${SERVER_TUNNEL_LOCAL_PORT}\"", "LESAVKA_TLS_DOMAIN=\"${LESAVKA_TLS_DOMAIN}\"", "tunneled to ${LESAVKA_SERVER_HOST}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT}", "CAPTURE_READY_MARKER=\"__LESAVKA_CAPTURE_READY__\"", "LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}", "LOCAL_REPORT_DIR=\"${LOCAL_OUTPUT_DIR%/}/lesavka-output-delay-probe-${STAMP}\"", "LOCAL_ANALYSIS_JSON=\"${LOCAL_REPORT_DIR}/report.json\"", "LOCAL_ANALYSIS_ERROR_LOG=\"${LOCAL_REPORT_DIR}/analysis-error.log\"", "LOCAL_EVENTS_CSV=\"${LOCAL_REPORT_DIR}/events.csv\"", "LOCAL_SERVER_PROBE_REPLY=\"${LOCAL_REPORT_DIR}/server-output-probe-reply.txt\"", "LOCAL_SERVER_TIMELINE_JSON=\"${LOCAL_REPORT_DIR}/server-output-timeline.json\"", "LOCAL_SIGNAL_CONDITIONING_REPLY=\"${LOCAL_REPORT_DIR}/signal-conditioning-probe-reply.txt\"", "LOCAL_SIGNAL_CONDITIONING_TIMELINE_JSON=\"${LOCAL_REPORT_DIR}/signal-conditioning-timeline.json\"", "LOCAL_OUTPUT_DELAY_CORRELATION_JSON=\"${LOCAL_REPORT_DIR}/output-delay-correlation.json\"", "LOCAL_OUTPUT_DELAY_CORRELATION_CSV=\"${LOCAL_REPORT_DIR}/output-delay-correlation.csv\"", "LOCAL_OUTPUT_DELAY_CORRELATION_TXT=\"${LOCAL_REPORT_DIR}/output-delay-correlation.txt\"", "LOCAL_CLOCK_ALIGNMENT_JSON=\"${LOCAL_REPORT_DIR}/clock-alignment.json\"", "LOCAL_OUTPUT_DELAY_JSON=\"${LOCAL_REPORT_DIR}/output-delay-calibration.json\"", "LOCAL_OUTPUT_DELAY_ENV=\"${LOCAL_REPORT_DIR}/output-delay-calibration.env\"", "LESAVKA_OUTPUT_DELAY_CALIBRATION=${LESAVKA_OUTPUT_DELAY_CALIBRATION:-1}", "LESAVKA_OUTPUT_DELAY_APPLY=${LESAVKA_OUTPUT_DELAY_APPLY:-0}", "LESAVKA_OUTPUT_DELAY_APPLY_MODE=${LESAVKA_OUTPUT_DELAY_APPLY_MODE:-absolute}", "LESAVKA_OUTPUT_DELAY_CONFIRM=${LESAVKA_OUTPUT_DELAY_CONFIRM:-1}", "LESAVKA_OUTPUT_DELAY_SAVE=${LESAVKA_OUTPUT_DELAY_SAVE:-0}", "LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=${LESAVKA_OUTPUT_REQUIRE_SYNC_PASS:-0}", "LESAVKA_OUTPUT_DELAY_TARGET=${LESAVKA_OUTPUT_DELAY_TARGET:-video}", "LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${LESAVKA_OUTPUT_DELAY_MIN_PAIRS:-13}", "LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS:-5000}", "LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS=${LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS:-80}", "LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_OUTPUT_DELAY_MAX_STEP_US:-1500000}", "LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS:-1000}", "LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS:-100}", "LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}", "LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=${LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS:-${LESAVKA_OUTPUT_DELAY_MIN_PAIRS}}", "REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS:-0}", "REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS:-1}", "remote_capture_ready_settle_seconds=${15}", "announce_capture_start", "signal_capture_ready", "run_tolerant_capture", "discarding %ss of post-enumeration capture before probe", "ffmpeg -nostdin -hide_banner", "\"$@\" UVC/UAC sinks -> lab host capture\"", "audio_after_video_positive", "active_audio_offset_us", "active_video_offset_us", "audio_target_offset_us = active_audio_offset_us + audio_delta_us", "video_target_offset_us = active_video_offset_us + video_delta_us", "output_delay_active_audio_offset_us", "output_delay_active_video_offset_us", "audio_target_offset_us", "video_target_offset_us", "output_delay_audio_target_offset_us", "output_delay_video_target_offset_us", "LESAVKA_OUTPUT_DELAY_CONFIRMING=1", "LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=1", "LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=\"${confirm_audio_delay}\"", "LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=\"${confirm_video_delay}\"", "==> confirming fixed UVC/UAC output-delay calibration", "required sync pass failed", "calibration_active_video_offset_us", "absolute_target_video_offset_us", "calibration_apply_video_delta_us", "PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}", "\"${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US}\"", "\"${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}\"", "REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}", "REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}", "REMOTE_ARTIFACT_RETRIES=${REMOTE_ARTIFACT_RETRIES:-3}", "REMOTE_ARTIFACT_RETRY_DELAY_SECONDS=${REMOTE_ARTIFACT_RETRY_DELAY_SECONDS:-5}", "retry_remote_artifact_command", "run_remote_sync_analysis_once", "checking remote capture on ${TETHYS_HOST}", "copying sync analyzer to ${TETHYS_HOST}", "running remote sync analysis on ${TETHYS_HOST}", "fetching capture from ${TETHYS_HOST}", "warning: failed to fetch capture artifact; continuing with remote analysis JSON when available", "analysis_tmp=\"${LOCAL_ANALYSIS_JSON}.tmp\"", "analysis_error_tmp=\"${LOCAL_ANALYSIS_ERROR_LOG}.tmp\"", "analysis_error_log: ${LOCAL_ANALYSIS_ERROR_LOG}", "event_visibility", "Coded event visibility", "video_only", "audio_only", "REMOTE_PULSE_AUDIO_ANCHOR_SILENCE=${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE:-1}", "anchoring Pulse capture audio timeline with generated silence", "audiotestsrc wave=silence is-live=true do-timestamp=true", "audiomixer name=amix ignore-inactive-pads=true", "output_delay_calibration_json", "direct UVC/UAC output-delay calibration", "calibration-save-default", "LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-0}", "PROBE_START_GRACE_SECONDS=${PROBE_START_GRACE_SECONDS:-20}", "PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + PROBE_START_GRACE_SECONDS))}", "CAPTURE_START_GRACE_SECONDS=${CAPTURE_START_GRACE_SECONDS:-12}", "CONDITIONING_CAPTURE_SECONDS=$((PROBE_CONDITIONING_SECONDS > 0 ? PROBE_CONDITIONING_SECONDS + PROBE_CONDITIONING_WARMUP_SECONDS + PROBE_CONDITIONING_GAP_SECONDS : 0))", "CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + CONDITIONING_CAPTURE_SECONDS + CAPTURE_START_GRACE_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))}", "ANALYSIS_TIMELINE_WINDOW=${ANALYSIS_TIMELINE_WINDOW:-0}", "compute_analysis_window_arg", "analyzer timeline window:", "run_output_delay_probe", "conditioning Tethys UVC/UAC signal path before measured probe", "signal_conditioning_reply: ${LOCAL_SIGNAL_CONDITIONING_REPLY}", "signal_conditioning_timeline_json: ${LOCAL_SIGNAL_CONDITIONING_TIMELINE_JSON}", "output-delay-probe", "server-generated UVC/UAC output-delay probe", "server output-delay probe timed out after ${PROBE_TIMEOUT_SECONDS}s", "--event-width-codes '${PROBE_EVENT_WIDTH_CODES}'", "VIDIOC_STREAMON.*Connection timed out", "the UVC host opened before MJPEG frames reached the gadget", "Tethys capture failed before the sync probe could start", "wait_for_capture_ready", "Timed out waiting for Tethys capture to become ready", "grep -q \"${CAPTURE_READY_MARKER}\"", "Lesavka UVC video device not found on Tethys; refusing to fall back to an unrelated webcam/capture card.", "resolve_alsa_audio_device", "PipeWire Lesavka source not found; using explicit diagnostic ALSA fallback device", "Set REMOTE_CAPTURE_STACK=alsa or REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=1 only for diagnostic signal-presence checks.", "Lesavka Pulse/PipeWire audio source not found; refusing raw ALSA fallback for timing-sensitive capture.", "discarding %ss of post-enumeration capture before probe", "using Pulse source:", "-f pulse", "-map 1:a:0", "artifact_dir: ${LOCAL_REPORT_DIR}", "events_csv: ${LOCAL_EVENTS_CSV}", "server_timeline_json: ${LOCAL_SERVER_TIMELINE_JSON}", "output_delay_correlation_json: ${LOCAL_OUTPUT_DELAY_CORRELATION_JSON}", "==> Lesavka versions under test", "lesavka-relayctl", "--bin lesavka-relayctl", "--bin lesavka-sync-analyze", "client_revision=", "server_version=", "server_revision=", "combined version+revision", ] { assert!( SYNC_SCRIPT.contains(expected), "manual sync script should contain {expected}" ); } assert!( !SYNC_SCRIPT.contains( "RESOLVED_LESAVKA_SERVER_ADDR=\"http://${LESAVKA_SERVER_CONNECT_HOST}:${port}\"" ), "auto server resolution should not guess a public gRPC host when SSH is already required" ); for forbidden in [ "LOCAL_AUDIO_SANITY", "run_local_audio_sanity.sh", "lesavka-sync-probe", ] { assert!( !SYNC_SCRIPT.contains(forbidden), "direct UVC/UAC output-delay probe must not run workstation/client-side probe behavior: {forbidden}" ); } } #[test] fn output_freshness_is_event_level_not_capture_container_duration() { for expected in [ "\"freshness_clock_starts_at\": \"server pipeline reference before intentional sync delay\"", "video_freshness_ms = (", "audio_freshness_ms = (", "\"video_freshness_stats\": video_freshness_stats", "\"audio_freshness_stats\": audio_freshness_stats", "\"worst_p95_pipeline_freshness_ms\": freshness_worst_p95_ms", "\"worst_event_age_with_uncertainty_ms\": freshness_worst_event_with_uncertainty_ms", "\"configured_capture_duration_s\": configured_capture_duration_s", "\"observed_capture_duration_s\": observed_capture_duration_s", "\"min_paired_events\": min_freshness_pairs", "\"paired_event_count\": len(joined)", "\"warnings\": capture_timebase_warnings", "treating whole-file media span as recorder coverage, not event freshness", "Freshness timeline", "measured event freshness", ] { assert!( SYNC_SCRIPT.contains(expected), "freshness contract should contain event-level timing marker {expected}" ); } let span_gap_start = SYNC_SCRIPT .find("span_gap_s = finite(media_timeline.get(\"audio_video_span_gap_s\"))") .expect("manual sync script should compute media span gap diagnostics"); let span_gap_end = SYNC_SCRIPT[span_gap_start..] .find("if capture_timebase_reasons:") .map(|offset| span_gap_start + offset) .expect("span gap diagnostic block should end before reason folding"); let span_gap_block = &SYNC_SCRIPT[span_gap_start..span_gap_end]; assert!( span_gap_block.contains("capture_timebase_warnings.append("), "whole-file media span mismatch should be recorded as a warning" ); assert!( !span_gap_block.contains("capture_timebase_status = \"invalid\""), "whole-file media span mismatch must not invalidate freshness; freshness is event-level" ); assert!( !span_gap_block.contains("capture_timebase_reasons.append("), "whole-file media span mismatch must not become a hard invalidation reason" ); } #[test] fn output_freshness_still_invalidates_real_event_timing_contradictions() { let negative_rows_start = SYNC_SCRIPT .find("elif video_negative_rows or audio_negative_rows:") .expect("manual sync script should detect observed-before-injection rows"); let negative_rows_end = SYNC_SCRIPT[negative_rows_start..] .find("span_gap_s = finite") .map(|offset| negative_rows_start + offset) .expect("negative-row invalidation should precede span diagnostics"); let negative_rows_block = &SYNC_SCRIPT[negative_rows_start..negative_rows_end]; for expected in [ "capture_timebase_status = \"invalid\"", "video observed before server pipeline injection", "audio observed before server pipeline injection", "capture_timebase_reasons.append(", ] { assert!( negative_rows_block.contains(expected), "real event timing contradictions must remain hard failures: {expected}" ); } for expected in [ "elif capture_timebase_status == \"invalid\":", "freshness_reason = f\"capture timebase invalid: {capture_timebase_reason}\"", "paired coded events {len(joined)} < {min_freshness_pairs}", "freshness requires enough matched injected/observed events", "server/capture clock alignment unavailable or too uncertain", "non-monotonic event timestamps", "minimum freshness", "freshness was negative beyond clock uncertainty", ] { assert!( SYNC_SCRIPT.contains(expected), "freshness should still fail on impossible event timing: {expected}" ); } }