lesavka/testing/tests/client_manual_sync_script_contract.rs

//! Contract tests for the manual upstream A/V sync harness.
//!
//! Scope: statically guard the workstation-side tunnel/bootstrap behavior.
//! Targets: `scripts/manual/run_upstream_av_sync.sh`.
//! Why: the manual probe should reach Theia through SSH even when the gRPC
//! port is not exposed on the public SSH endpoint.

const SYNC_SCRIPT: &str = include_str!("../../scripts/manual/run_upstream_av_sync.sh");
#[test]
fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
    for expected in [
        "cleanup_server_tunnel",
        "pick_local_server_tunnel_port",
        "wait_for_server_tunnel",
        "start_server_tunnel",
        "ExitOnForwardFailure=yes",
        "127.0.0.1:${local_port}:127.0.0.1:${remote_port}",
        "LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https}",
        "LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}",
        "RESOLVED_LESAVKA_SERVER_ADDR=\"${LESAVKA_SERVER_SCHEME}://127.0.0.1:${SERVER_TUNNEL_LOCAL_PORT}\"",
        "LESAVKA_TLS_DOMAIN=\"${LESAVKA_TLS_DOMAIN}\"",
        "tunneled to ${LESAVKA_SERVER_HOST}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT}",
        "CAPTURE_READY_MARKER=\"__LESAVKA_CAPTURE_READY__\"",
        "LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}",
        "LOCAL_REPORT_DIR=\"${LOCAL_OUTPUT_DIR%/}/lesavka-output-delay-probe-${STAMP}\"",
        "LOCAL_ANALYSIS_JSON=\"${LOCAL_REPORT_DIR}/report.json\"",
        "LOCAL_ANALYSIS_ERROR_LOG=\"${LOCAL_REPORT_DIR}/analysis-error.log\"",
        "LOCAL_EVENTS_CSV=\"${LOCAL_REPORT_DIR}/events.csv\"",
        "LOCAL_SERVER_PROBE_REPLY=\"${LOCAL_REPORT_DIR}/server-output-probe-reply.txt\"",
        "LOCAL_SERVER_TIMELINE_JSON=\"${LOCAL_REPORT_DIR}/server-output-timeline.json\"",
        "LOCAL_SIGNAL_CONDITIONING_REPLY=\"${LOCAL_REPORT_DIR}/signal-conditioning-probe-reply.txt\"",
        "LOCAL_SIGNAL_CONDITIONING_TIMELINE_JSON=\"${LOCAL_REPORT_DIR}/signal-conditioning-timeline.json\"",
        "LOCAL_OUTPUT_DELAY_CORRELATION_JSON=\"${LOCAL_REPORT_DIR}/output-delay-correlation.json\"",
        "LOCAL_OUTPUT_DELAY_CORRELATION_CSV=\"${LOCAL_REPORT_DIR}/output-delay-correlation.csv\"",
        "LOCAL_OUTPUT_DELAY_CORRELATION_TXT=\"${LOCAL_REPORT_DIR}/output-delay-correlation.txt\"",
        "LOCAL_CLOCK_ALIGNMENT_JSON=\"${LOCAL_REPORT_DIR}/clock-alignment.json\"",
        "LOCAL_OUTPUT_DELAY_JSON=\"${LOCAL_REPORT_DIR}/output-delay-calibration.json\"",
        "LOCAL_OUTPUT_DELAY_ENV=\"${LOCAL_REPORT_DIR}/output-delay-calibration.env\"",
        "LESAVKA_OUTPUT_DELAY_CALIBRATION=${LESAVKA_OUTPUT_DELAY_CALIBRATION:-1}",
        "LESAVKA_OUTPUT_DELAY_APPLY=${LESAVKA_OUTPUT_DELAY_APPLY:-0}",
        "LESAVKA_OUTPUT_DELAY_APPLY_MODE=${LESAVKA_OUTPUT_DELAY_APPLY_MODE:-absolute}",
        "LESAVKA_OUTPUT_DELAY_CONFIRM=${LESAVKA_OUTPUT_DELAY_CONFIRM:-1}",
        "LESAVKA_OUTPUT_DELAY_SAVE=${LESAVKA_OUTPUT_DELAY_SAVE:-0}",
        "LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=${LESAVKA_OUTPUT_REQUIRE_SYNC_PASS:-0}",
        "LESAVKA_OUTPUT_DELAY_TARGET=${LESAVKA_OUTPUT_DELAY_TARGET:-video}",
        "LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${LESAVKA_OUTPUT_DELAY_MIN_PAIRS:-13}",
        "LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS:-5000}",
        "LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS=${LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS:-80}",
        "LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_OUTPUT_DELAY_MAX_STEP_US:-1500000}",
        "LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS:-1000}",
        "LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS:-100}",
        "LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}",
        "LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=${LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS:-${LESAVKA_OUTPUT_DELAY_MIN_PAIRS}}",
        "REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS:-0}",
        "REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS:-1}",
        "remote_capture_ready_settle_seconds=${15}",
        "announce_capture_start",
        "signal_capture_ready",
        "run_tolerant_capture",
        "discarding %ss of post-enumeration capture before probe",
        "ffmpeg -nostdin -hide_banner",
        "\"$@\" </dev/null",
        "timeout --kill-after=5 --signal=INT",
        "timeout --kill-after=3 --signal=INT",
        "REMOTE_EXPECT_UVC_WIDTH=${REMOTE_EXPECT_UVC_WIDTH:-}",
        "REMOTE_EXPECT_UVC_HEIGHT=${REMOTE_EXPECT_UVC_HEIGHT:-}",
        "REMOTE_EXPECT_UVC_FPS=${REMOTE_EXPECT_UVC_FPS:-}",
        "server.env UVC_MODE=",
        "uvc.env    UVC_MODE=",
        "effective UVC_MODE=",
        "expected effective UVC_WIDTH",
        "expected uvc.env UVC_FPS",
        "server-to-capture clock alignment unavailable; falling back to client-mediated SSH samples",
        "LESAVKA_CLOCK_ALIGNMENT_SAMPLES=${LESAVKA_CLOCK_ALIGNMENT_SAMPLES:-5}",
        "sample_best_host_clock_offset_ns",
        "sample_server_to_capture_clock_offset_ns",
        "persistent-ssh-python",
        "server-to-capture-persistent-ssh-python",
        "LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}",
        "LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US:-0}",
        "PROBE_CONDITIONING_SECONDS=${PROBE_CONDITIONING_SECONDS:-0}",
        "PROBE_CONDITIONING_WARMUP_SECONDS=${PROBE_CONDITIONING_WARMUP_SECONDS:-1}",
        "PROBE_CONDITIONING_GAP_SECONDS=${PROBE_CONDITIONING_GAP_SECONDS:-1}",
        "write_clock_alignment",
        "capture_start_unix_ns=",
        "write_output_delay_calibration",
        "extract_timeline_from_reply",
        "extract_server_timeline",
        "extract_signal_conditioning_timeline",
        "write_output_delay_correlation",
        "maybe_apply_output_delay_calibration",
        "maybe_run_output_delay_confirmation",
        "enforce_sync_verdict",
        "schema\": \"lesavka.output-delay-calibration.v1\"",
        "schema\": \"lesavka.output-delay-correlation.v1\"",
        "schema\": \"lesavka.clock-alignment.v1\"",
        "schema\": \"lesavka.output-freshness-summary.v1\"",
        "schema\": \"lesavka.output-timing-map.v1\"",
        "server_timeline_json=",
        "clock_alignment_json: ${LOCAL_CLOCK_ALIGNMENT_JSON}",
        "dominant_layer",
        "Timing map",
        "capture to server probe start",
        "setup/pre-roll, not media latency",
        "injection point",
        "server-final-output-handoff",
        "video CameraRelay::feed; audio Voice::push",
        "client uplink included",
        "freshness clock starts",
        "freshness status",
        "clock-corrected server pipeline reference to Tethys observed playback",
        "capture timebase",
        "capture timebase invalid",
        "capture_timebase_warnings = []",
        "capture timebase warning",
        "treating whole-file media span as recorder coverage, not event freshness",
        "intentional sync delays",
        "sink handoff overhead",
        "pipeline freshness",
        "Freshness timeline",
        "capture duration",
        "observed_capture_duration_s",
        "configured_capture_duration_s",
        "recorder coverage, not media latency",
        "active probe media",
        "setup before probe",
        "measured event freshness",
        "freshness verdict",
        "freshness budget",
        "Output smoothness",
        "capture media timeline",
        "media_timeline",
        "video continuity",
        "audio pilot continuity",
        "schema\": \"lesavka.output-smoothness-summary.v1\"",
        "video_freshness_ms",
        "audio_freshness_ms",
        "video_event_age_ms",
        "audio_event_age_ms",
        "intentional_video_delay_ms",
        "video_event_age_stats",
        "video_sink_handoff_overhead_stats",
        "minimum_pipeline_freshness_ms",
        "worst_event_age_with_uncertainty_ms",
        "minimum_event_age_ms",
        "min_paired_events",
        "paired_event_count",
        "non_monotonic_event_timestamps",
        "sync_passed",
        "sync did not pass, so freshness from paired signatures is not trustworthy",
        "impossible negative pipeline freshness",
        "video_delay_function_candidate",
        "source\": \"direct-uvc-uac-output-probe\"",
        "scope\": \"server-output-static-baseline\"",
        "applies_to\": \"server UVC/UAC gadget output path\"",
        "measurement_host_role\": \"lab-attached USB host\"",
        "injection_scope\": \"server-final-output-handoff\"",
        "client_uplink_included\": False",
        "probe_media_origin\": \"server-generated\"",
        "probe_media_path\": \"server generated signatures -> UVC/UAC sinks -> lab host capture\"",
        "audio_after_video_positive",
        "active_audio_offset_us",
        "active_video_offset_us",
        "audio_target_offset_us = active_audio_offset_us + audio_delta_us",
        "video_target_offset_us = active_video_offset_us + video_delta_us",
        "output_delay_active_audio_offset_us",
        "output_delay_active_video_offset_us",
        "audio_target_offset_us",
        "video_target_offset_us",
        "output_delay_audio_target_offset_us",
        "output_delay_video_target_offset_us",
        "LESAVKA_OUTPUT_DELAY_CONFIRMING=1",
        "LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=1",
        "LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=\"${confirm_audio_delay}\"",
        "LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=\"${confirm_video_delay}\"",
        "==> confirming fixed UVC/UAC output-delay calibration",
        "required sync pass failed",
        "calibration_active_video_offset_us",
        "absolute_target_video_offset_us",
        "calibration_apply_video_delta_us",
        "PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}",
        "\"${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US}\"",
        "\"${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}\"",
        "REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}",
        "REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}",
        "REMOTE_ARTIFACT_RETRIES=${REMOTE_ARTIFACT_RETRIES:-3}",
        "REMOTE_ARTIFACT_RETRY_DELAY_SECONDS=${REMOTE_ARTIFACT_RETRY_DELAY_SECONDS:-5}",
        "retry_remote_artifact_command",
        "run_remote_sync_analysis_once",
        "checking remote capture on ${TETHYS_HOST}",
        "copying sync analyzer to ${TETHYS_HOST}",
        "running remote sync analysis on ${TETHYS_HOST}",
        "fetching capture from ${TETHYS_HOST}",
        "warning: failed to fetch capture artifact; continuing with remote analysis JSON when available",
        "analysis_tmp=\"${LOCAL_ANALYSIS_JSON}.tmp\"",
        "analysis_error_tmp=\"${LOCAL_ANALYSIS_ERROR_LOG}.tmp\"",
        "analysis_error_log: ${LOCAL_ANALYSIS_ERROR_LOG}",
        "event_visibility",
        "Coded event visibility",
        "video_only",
        "audio_only",
        "REMOTE_PULSE_AUDIO_ANCHOR_SILENCE=${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE:-1}",
        "anchoring Pulse capture audio timeline with generated silence",
        "audiotestsrc wave=silence is-live=true do-timestamp=true",
        "audiomixer name=amix ignore-inactive-pads=true",
        "output_delay_calibration_json",
        "direct UVC/UAC output-delay calibration",
        "calibration-save-default",
        "LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-0}",
        "PROBE_START_GRACE_SECONDS=${PROBE_START_GRACE_SECONDS:-20}",
        "PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + PROBE_START_GRACE_SECONDS))}",
        "CAPTURE_START_GRACE_SECONDS=${CAPTURE_START_GRACE_SECONDS:-12}",
        "CONDITIONING_CAPTURE_SECONDS=$((PROBE_CONDITIONING_SECONDS > 0 ? PROBE_CONDITIONING_SECONDS + PROBE_CONDITIONING_WARMUP_SECONDS + PROBE_CONDITIONING_GAP_SECONDS : 0))",
        "CAPTURE_SECONDS=${CAPTURE_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + CONDITIONING_CAPTURE_SECONDS + CAPTURE_START_GRACE_SECONDS + LEAD_IN_SECONDS + TAIL_SECONDS))}",
        "ANALYSIS_TIMELINE_WINDOW=${ANALYSIS_TIMELINE_WINDOW:-0}",
        "compute_analysis_window_arg",
        "analyzer timeline window:",
        "run_output_delay_probe",
        "conditioning Tethys UVC/UAC signal path before measured probe",
        "signal_conditioning_reply: ${LOCAL_SIGNAL_CONDITIONING_REPLY}",
        "signal_conditioning_timeline_json: ${LOCAL_SIGNAL_CONDITIONING_TIMELINE_JSON}",
        "output-delay-probe",
        "server-generated UVC/UAC output-delay probe",
        "server output-delay probe timed out after ${PROBE_TIMEOUT_SECONDS}s",
        "--event-width-codes '${PROBE_EVENT_WIDTH_CODES}'",
        "VIDIOC_STREAMON.*Connection timed out",
        "the UVC host opened before MJPEG frames reached the gadget",
        "Tethys capture failed before the sync probe could start",
        "wait_for_capture_ready",
        "Timed out waiting for Tethys capture to become ready",
        "grep -q \"${CAPTURE_READY_MARKER}\"",
        "Lesavka UVC video device not found on Tethys; refusing to fall back to an unrelated webcam/capture card.",
        "resolve_alsa_audio_device",
        "PipeWire Lesavka source not found; using explicit diagnostic ALSA fallback device",
        "Set REMOTE_CAPTURE_STACK=alsa or REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=1 only for diagnostic signal-presence checks.",
        "Lesavka Pulse/PipeWire audio source not found; refusing raw ALSA fallback for timing-sensitive capture.",
        "discarding %ss of post-enumeration capture before probe",
        "using Pulse source:",
        "-f pulse",
        "-map 1:a:0",
        "artifact_dir: ${LOCAL_REPORT_DIR}",
        "events_csv: ${LOCAL_EVENTS_CSV}",
        "server_timeline_json: ${LOCAL_SERVER_TIMELINE_JSON}",
        "output_delay_correlation_json: ${LOCAL_OUTPUT_DELAY_CORRELATION_JSON}",
        "==> Lesavka versions under test",
        "lesavka-relayctl",
        "--bin lesavka-relayctl",
        "--bin lesavka-sync-analyze",
        "client_revision=",
        "server_version=",
        "server_revision=",
        "combined version+revision",
    ] {
        assert!(
            SYNC_SCRIPT.contains(expected),
            "manual sync script should contain {expected}"
        );
    }
    assert!(
        !SYNC_SCRIPT.contains(
            "RESOLVED_LESAVKA_SERVER_ADDR=\"http://${LESAVKA_SERVER_CONNECT_HOST}:${port}\""
        ),
        "auto server resolution should not guess a public gRPC host when SSH is already required"
    );
    for forbidden in [
        "LOCAL_AUDIO_SANITY",
        "run_local_audio_sanity.sh",
        "lesavka-sync-probe",
    ] {
        assert!(
            !SYNC_SCRIPT.contains(forbidden),
            "direct UVC/UAC output-delay probe must not run workstation/client-side probe behavior: {forbidden}"
        );
    }
}

#[test]
fn output_freshness_is_event_level_not_capture_container_duration() {
    for expected in [
        "\"freshness_clock_starts_at\": \"server pipeline reference before intentional sync delay\"",
        "video_freshness_ms = (",
        "audio_freshness_ms = (",
        "\"video_freshness_stats\": video_freshness_stats",
        "\"audio_freshness_stats\": audio_freshness_stats",
        "\"worst_p95_pipeline_freshness_ms\": freshness_worst_p95_ms",
        "\"worst_event_age_with_uncertainty_ms\": freshness_worst_event_with_uncertainty_ms",
        "\"configured_capture_duration_s\": configured_capture_duration_s",
        "\"observed_capture_duration_s\": observed_capture_duration_s",
        "\"min_paired_events\": min_freshness_pairs",
        "\"paired_event_count\": len(joined)",
        "\"warnings\": capture_timebase_warnings",
        "treating whole-file media span as recorder coverage, not event freshness",
        "Freshness timeline",
        "measured event freshness",
    ] {
        assert!(
            SYNC_SCRIPT.contains(expected),
            "freshness contract should contain event-level timing marker {expected}"
        );
    }

    let span_gap_start = SYNC_SCRIPT
        .find("span_gap_s = finite(media_timeline.get(\"audio_video_span_gap_s\"))")
        .expect("manual sync script should compute media span gap diagnostics");
    let span_gap_end = SYNC_SCRIPT[span_gap_start..]
        .find("if capture_timebase_reasons:")
        .map(|offset| span_gap_start + offset)
        .expect("span gap diagnostic block should end before reason folding");
    let span_gap_block = &SYNC_SCRIPT[span_gap_start..span_gap_end];

    assert!(
        span_gap_block.contains("capture_timebase_warnings.append("),
        "whole-file media span mismatch should be recorded as a warning"
    );
    assert!(
        !span_gap_block.contains("capture_timebase_status = \"invalid\""),
        "whole-file media span mismatch must not invalidate freshness; freshness is event-level"
    );
    assert!(
        !span_gap_block.contains("capture_timebase_reasons.append("),
        "whole-file media span mismatch must not become a hard invalidation reason"
    );
}

#[test]
fn output_freshness_still_invalidates_real_event_timing_contradictions() {
    let negative_rows_start = SYNC_SCRIPT
        .find("elif video_negative_rows or audio_negative_rows:")
        .expect("manual sync script should detect observed-before-injection rows");
    let negative_rows_end = SYNC_SCRIPT[negative_rows_start..]
        .find("span_gap_s = finite")
        .map(|offset| negative_rows_start + offset)
        .expect("negative-row invalidation should precede span diagnostics");
    let negative_rows_block = &SYNC_SCRIPT[negative_rows_start..negative_rows_end];

    for expected in [
        "capture_timebase_status = \"invalid\"",
        "video observed before server pipeline injection",
        "audio observed before server pipeline injection",
        "capture_timebase_reasons.append(",
    ] {
        assert!(
            negative_rows_block.contains(expected),
            "real event timing contradictions must remain hard failures: {expected}"
        );
    }

    for expected in [
        "elif capture_timebase_status == \"invalid\":",
        "freshness_reason = f\"capture timebase invalid: {capture_timebase_reason}\"",
        "paired coded events {len(joined)} < {min_freshness_pairs}",
        "freshness requires enough matched injected/observed events",
        "server/capture clock alignment unavailable or too uncertain",
        "non-monotonic event timestamps",
        "minimum freshness",
        "freshness was negative beyond clock uncertainty",
    ] {
        assert!(
            SYNC_SCRIPT.contains(expected),
            "freshness should still fail on impossible event timing: {expected}"
        );
    }
}