lesavka/testing/tests/client_manual_sync_script_contract.rs

//! Contract tests for the manual upstream A/V sync harness.
//!
//! Scope: statically guard the workstation-side tunnel/bootstrap behavior.
//! Targets: `scripts/manual/run_upstream_av_sync.sh`.
//! Why: the manual probe should reach Theia through SSH even when the gRPC
//! port is not exposed on the public SSH endpoint.

const SYNC_SCRIPT: &str = include_str!("../../scripts/manual/run_upstream_av_sync.sh");
const BROWSER_SYNC_SCRIPT: &str =
    include_str!("../../scripts/manual/run_upstream_browser_av_sync.sh");
const MIRRORED_SYNC_SCRIPT: &str =
    include_str!("../../scripts/manual/run_upstream_mirrored_av_sync.sh");
const LOCAL_STIMULUS: &str = include_str!("../../scripts/manual/local_av_stimulus.py");
const SYNC_PROBE_RUNNER: &str = include_str!("../../client/src/sync_probe/runner.rs");

#[test]
fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
    for expected in [
        "cleanup_server_tunnel",
        "pick_local_server_tunnel_port",
        "wait_for_server_tunnel",
        "start_server_tunnel",
        "ExitOnForwardFailure=yes",
        "127.0.0.1:${local_port}:127.0.0.1:${remote_port}",
        "LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME:-https}",
        "LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN:-lesavka-server}",
        "RESOLVED_LESAVKA_SERVER_ADDR=\"${LESAVKA_SERVER_SCHEME}://127.0.0.1:${SERVER_TUNNEL_LOCAL_PORT}\"",
        "LESAVKA_TLS_DOMAIN=\"${LESAVKA_TLS_DOMAIN}\"",
        "tunneled to ${LESAVKA_SERVER_HOST}:127.0.0.1:${SERVER_TUNNEL_REMOTE_PORT}",
        "CAPTURE_READY_MARKER=\"__LESAVKA_CAPTURE_READY__\"",
        "LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}",
        "LOCAL_REPORT_DIR=\"${LOCAL_OUTPUT_DIR%/}/lesavka-output-delay-probe-${STAMP}\"",
        "LOCAL_ANALYSIS_JSON=\"${LOCAL_REPORT_DIR}/report.json\"",
        "LOCAL_EVENTS_CSV=\"${LOCAL_REPORT_DIR}/events.csv\"",
        "LOCAL_SERVER_PROBE_REPLY=\"${LOCAL_REPORT_DIR}/server-output-probe-reply.txt\"",
        "LOCAL_SERVER_TIMELINE_JSON=\"${LOCAL_REPORT_DIR}/server-output-timeline.json\"",
        "LOCAL_OUTPUT_DELAY_CORRELATION_JSON=\"${LOCAL_REPORT_DIR}/output-delay-correlation.json\"",
        "LOCAL_OUTPUT_DELAY_CORRELATION_CSV=\"${LOCAL_REPORT_DIR}/output-delay-correlation.csv\"",
        "LOCAL_OUTPUT_DELAY_CORRELATION_TXT=\"${LOCAL_REPORT_DIR}/output-delay-correlation.txt\"",
        "LOCAL_OUTPUT_DELAY_JSON=\"${LOCAL_REPORT_DIR}/output-delay-calibration.json\"",
        "LOCAL_OUTPUT_DELAY_ENV=\"${LOCAL_REPORT_DIR}/output-delay-calibration.env\"",
        "LESAVKA_OUTPUT_DELAY_CALIBRATION=${LESAVKA_OUTPUT_DELAY_CALIBRATION:-1}",
        "LESAVKA_OUTPUT_DELAY_APPLY=${LESAVKA_OUTPUT_DELAY_APPLY:-0}",
        "LESAVKA_OUTPUT_DELAY_SAVE=${LESAVKA_OUTPUT_DELAY_SAVE:-0}",
        "LESAVKA_OUTPUT_DELAY_TARGET=${LESAVKA_OUTPUT_DELAY_TARGET:-video}",
        "LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${LESAVKA_OUTPUT_DELAY_MIN_PAIRS:-8}",
        "LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS:-5000}",
        "LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS=${LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS:-80}",
        "LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_OUTPUT_DELAY_MAX_STEP_US:-1500000}",
        "LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}",
        "LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US:-0}",
        "write_output_delay_calibration",
        "extract_server_timeline",
        "write_output_delay_correlation",
        "maybe_apply_output_delay_calibration",
        "schema\": \"lesavka.output-delay-calibration.v1\"",
        "schema\": \"lesavka.output-delay-correlation.v1\"",
        "server_timeline_json=",
        "dominant_layer",
        "video_delay_function_candidate",
        "source\": \"direct-uvc-uac-output-probe\"",
        "scope\": \"server-output-static-baseline\"",
        "applies_to\": \"server UVC/UAC gadget output path\"",
        "measurement_host_role\": \"lab-attached USB host\"",
        "probe_media_origin\": \"server-generated\"",
        "probe_media_path\": \"server generated signatures -> UVC/UAC sinks -> lab host capture\"",
        "audio_after_video_positive",
        "PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES:-1,2,1,3",
        "\"${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US}\"",
        "\"${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}\"",
        "REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}",
        "output_delay_calibration_json",
        "direct UVC/UAC output-delay calibration",
        "calibration-save-default",
        "LEAD_IN_SECONDS=${LEAD_IN_SECONDS:-0}",
        "PROBE_TIMEOUT_SECONDS=${PROBE_TIMEOUT_SECONDS:-$((PROBE_DURATION_SECONDS + PROBE_WARMUP_SECONDS + 20))}",
        "output-delay-probe",
        "server-generated UVC/UAC output-delay probe",
        "server output-delay probe timed out after ${PROBE_TIMEOUT_SECONDS}s",
        "--event-width-codes '${PROBE_EVENT_WIDTH_CODES}'",
        "VIDIOC_STREAMON.*Connection timed out",
        "the UVC host opened before MJPEG frames reached the gadget",
        "Tethys capture failed before the sync probe could start",
        "wait_for_capture_ready",
        "Timed out waiting for Tethys capture to become ready",
        "grep -q \"${CAPTURE_READY_MARKER}\"",
        "Lesavka UVC video device not found on Tethys; refusing to fall back to an unrelated webcam/capture card.",
        "resolve_alsa_audio_device",
        "PipeWire Lesavka source not found; falling back to ALSA device",
        "Lesavka audio source not found in PipeWire or ALSA; capture host does not currently expose the gadget microphone.",
        "artifact_dir: ${LOCAL_REPORT_DIR}",
        "events_csv: ${LOCAL_EVENTS_CSV}",
        "server_timeline_json: ${LOCAL_SERVER_TIMELINE_JSON}",
        "output_delay_correlation_json: ${LOCAL_OUTPUT_DELAY_CORRELATION_JSON}",
        "==> Lesavka versions under test",
        "lesavka-relayctl",
        "--bin lesavka-relayctl",
        "--bin lesavka-sync-analyze",
        "client_revision=",
        "server_version=",
        "server_revision=",
        "combined version+revision",
    ] {
        assert!(
            SYNC_SCRIPT.contains(expected),
            "manual sync script should contain {expected}"
        );
    }
    assert!(
        !SYNC_SCRIPT.contains(
            "RESOLVED_LESAVKA_SERVER_ADDR=\"http://${LESAVKA_SERVER_CONNECT_HOST}:${port}\""
        ),
        "auto server resolution should not guess a public gRPC host when SSH is already required"
    );
    for forbidden in [
        "LOCAL_AUDIO_SANITY",
        "run_local_audio_sanity.sh",
        "lesavka-sync-probe",
    ] {
        assert!(
            !SYNC_SCRIPT.contains(forbidden),
            "direct UVC/UAC output-delay probe must not run workstation/client-side probe behavior: {forbidden}"
        );
    }
}

#[test]
fn browser_sync_script_can_delegate_to_a_real_path_driver() {
    for expected in [
        "BROWSER_RECORD_SECONDS=${BROWSER_RECORD_SECONDS:-${PROBE_DURATION_SECONDS}}",
        "BROWSER_SYNC_DRIVER_COMMAND=${BROWSER_SYNC_DRIVER_COMMAND:-}",
        "BROWSER_CONSUMER_REUSE_SESSION=${BROWSER_CONSUMER_REUSE_SESSION:-0}",
        "BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED:-1}",
        "SYNC_ANALYZE_EVENT_WIDTH_CODES=${SYNC_ANALYZE_EVENT_WIDTH_CODES:-}",
        "==> running custom browser sync driver",
        "bash -lc \"${BROWSER_SYNC_DRIVER_COMMAND}\"",
        "browser_start_token=${browser_start_token}",
        "uploaded_start_token",
        "BROWSER_START_TOKEN",
        "analysis-failure.json",
        "BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED}",
        "BROWSER_START_ATTEMPTS=${BROWSER_START_ATTEMPTS:-5}",
        "browser consumer start attempt ${attempt}/${BROWSER_START_ATTEMPTS} failed; retrying",
        "--event-width-codes",
        "--report-dir \"${LOCAL_REPORT_DIR}\"",
        "for attempt in 1 2 3 4 5",
        "capture fetch attempt ${attempt} failed; retrying",
        "failed to fetch browser capture from ${TETHYS_HOST}:${REMOTE_CAPTURE}",
        r"raw activity delta was ([+-]?[0-9]+(?:\.[0-9]+)?) ms ",
        r"\(video=([0-9]+(?:\.[0-9]+)?)s audio=([0-9]+(?:\.[0-9]+)?)s\)",
    ] {
        assert!(
            BROWSER_SYNC_SCRIPT.contains(expected),
            "browser sync script should contain {expected}"
        );
    }
    assert!(
        !BROWSER_SYNC_SCRIPT.contains(r"(?:\\.[0-9]+)?"),
        "browser sync raw-delta parser should not require a literal backslash before decimals"
    );
}

#[test]
fn sync_probe_runner_uses_bundled_webcam_media_path() {
    for expected in [
        "bundled_webcam_media",
        "refusing to measure split upstream",
        "UpstreamMediaBundle",
        "stream_webcam_media",
        "PROBE_BUNDLE_SESSION_ID",
        "PROBE_BUNDLE_AUDIO_GRACE",
        "server does not advertise bundled webcam media",
    ] {
        assert!(
            SYNC_PROBE_RUNNER.contains(expected),
            "sync probe runner should contain {expected}"
        );
    }
    for forbidden in [
        ".stream_camera(Request::new(outbound))",
        ".stream_microphone(Request::new(outbound))",
    ] {
        assert!(
            !SYNC_PROBE_RUNNER.contains(forbidden),
            "sync probe runner must not use old split upstream RPC {forbidden}"
        );
    }
}

#[test]
fn mirrored_sync_script_uses_real_client_capture_path() {
    for expected in [
        "local_av_stimulus.py",
        "lesavka-client",
        "LESAVKA_HEADLESS=1",
        "LESAVKA_MEDIA_CONTROL=\"${MEDIA_CONTROL}\"",
        "LESAVKA_REQUIRE_EXPLICIT_MEDIA_SOURCES=\"${LESAVKA_REQUIRE_EXPLICIT_MEDIA_SOURCES:-1}\"",
        "--no-launcher --server \"${RESOLVED_LESAVKA_SERVER_ADDR}\"",
        "BROWSER_SYNC_DRIVER_COMMAND=\"${driver_command}\"",
        "SYNC_ANALYZE_EVENT_WIDTH_CODES=\"${PROBE_EVENT_WIDTH_CODES}\"",
        "run_upstream_browser_av_sync.sh",
        "wait_for_stimulus_page_ready 15",
        "Point the real webcam at the stimulus window",
        "==> Lesavka versions under test",
        "lesavka-relayctl",
        "--bin lesavka-relayctl",
        "client_revision=",
        "server_version=",
        "server_revision=",
        "combined version+revision",
        "run_status=0",
        "run_mirrored_segments || run_status=$?",
        "LESAVKA_SYNC_APPLY_CALIBRATION=${LESAVKA_SYNC_APPLY_CALIBRATION:-0}",
        "LESAVKA_SYNC_SAVE_CALIBRATION=${LESAVKA_SYNC_SAVE_CALIBRATION:-0}",
        "LESAVKA_SYNC_CALIBRATION_TARGET=${LESAVKA_SYNC_CALIBRATION_TARGET:-video}",
        "LESAVKA_SYNC_ADAPTIVE_CALIBRATION=${LESAVKA_SYNC_ADAPTIVE_CALIBRATION:-0}",
        "LESAVKA_SYNC_CALIBRATION_SEGMENTS=${LESAVKA_SYNC_CALIBRATION_SEGMENTS:-1}",
        "LESAVKA_SYNC_CONTINUOUS_BROWSER=${LESAVKA_SYNC_CONTINUOUS_BROWSER:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
        "LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE=${LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
        "LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS=${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS:-3}",
        "PROBE_AUDIO_GAIN=${PROBE_AUDIO_GAIN:-0.55}",
        "LESAVKA_STIMULUS_PREVIEW_SECONDS=${LESAVKA_STIMULUS_PREVIEW_SECONDS:-4}",
        "LESAVKA_OPEN_MANUAL_REVIEW_DOLPHIN=${LESAVKA_OPEN_MANUAL_REVIEW_DOLPHIN:-1}",
        "LESAVKA_SYNC_PROVISIONAL_CALIBRATION=${LESAVKA_SYNC_PROVISIONAL_CALIBRATION:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
        "LESAVKA_SYNC_PROVISIONAL_MIN_PAIRS=${LESAVKA_SYNC_PROVISIONAL_MIN_PAIRS:-3}",
        "LESAVKA_SYNC_PROVISIONAL_MAX_P95_MS=${LESAVKA_SYNC_PROVISIONAL_MAX_P95_MS:-350}",
        "LESAVKA_SYNC_PROVISIONAL_MAX_DRIFT_MS=${LESAVKA_SYNC_PROVISIONAL_MAX_DRIFT_MS:-250}",
        "LESAVKA_SYNC_PROVISIONAL_GAIN=${LESAVKA_SYNC_PROVISIONAL_GAIN:-0.5}",
        "LESAVKA_SYNC_PROVISIONAL_MAX_STEP_US=${LESAVKA_SYNC_PROVISIONAL_MAX_STEP_US:-150000}",
        "LESAVKA_SYNC_RAW_FAILURE_CALIBRATION=${LESAVKA_SYNC_RAW_FAILURE_CALIBRATION:-0}",
        "LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS=${LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS:-3}",
        "LESAVKA_SYNC_RAW_FAILURE_MAX_ABS_DELTA_MS=${LESAVKA_SYNC_RAW_FAILURE_MAX_ABS_DELTA_MS:-350}",
        "LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION=${LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
        "LESAVKA_SYNC_CONFIRMATION_SEGMENTS=${LESAVKA_SYNC_CONFIRMATION_SEGMENTS:-1}",
        "LESAVKA_SYNC_REQUIRE_CONFIRMATION_PASS=${LESAVKA_SYNC_REQUIRE_CONFIRMATION_PASS:-${LESAVKA_SYNC_CONFIRM_AFTER_CALIBRATION}}",
        "LESAVKA_SYNC_CONFIRMATION_SEGMENTS must be a non-negative integer",
        "LESAVKA_SYNC_TOTAL_SEGMENTS=$((LESAVKA_SYNC_CALIBRATION_SEGMENTS + LESAVKA_SYNC_CONFIRMATION_SEGMENTS))",
        "export LESAVKA_SYNC_PROVISIONAL_CALIBRATION",
        "export LESAVKA_SYNC_RAW_FAILURE_CALIBRATION",
        "export LESAVKA_SYNC_RAW_FAILURE_MIN_PAIRS",
        "LESAVKA_SYNC_ADAPTIVE_CALIBRATION",
        "LESAVKA_SYNC_CALIBRATION_SEGMENTS=4",
        "browser_consumer_reuse_session=${reuse_browser_session}",
        "browser_analysis_required=${analysis_required}",
        "BROWSER_CONSUMER_REUSE_SESSION=\"${reuse_browser_session}\"",
        "BROWSER_ANALYSIS_REQUIRED=\"${analysis_required}\"",
        "--audio-gain \"${PROBE_AUDIO_GAIN}\"",
        "run_stimulus_preview",
        "write_stimulus_driver_script",
        "local_stimulus_started=true",
        "observed_start_token",
        "audio_state",
        "LESAVKA_SYNC_CALIBRATION_SEGMENTS must be a positive integer",
        "run_mirrored_segments",
        "summarize_adaptive_probe_metrics",
        "for segment in $(seq 1 \"${LESAVKA_SYNC_TOTAL_SEGMENTS}\")",
        "segment_phase",
        "confirmation segment: calibration apply disabled so this segment tests the active calibration",
        "segment-${segment}",
        "calibration-before.env",
        "planner-before.env",
        "calibration-decision.env",
        "segment-metrics.csv",
        "segment-metrics.jsonl",
        "segment-events.csv",
        "segment-events.jsonl",
        "manual-review",
        "manual_review_html",
        "manual_review_dir",
        "open_manual_review_in_dolphin",
        "dolphin",
        "capture_path",
        "confirmation-summary.json",
        "confirmation_passed",
        "check_confirmation_result",
        "confirmation check failed",
        "analysis_failure_reason",
        "probe_activity_start_delta_ms",
        "blind-targets.json",
        "no segment produced a passing probe verdict; refusing to invent blind targets",
        "confirmation did not pass; refusing to promote calibration-only segments to blind targets",
        "candidate_good_calibration_segments",
        "decision_mode",
        "decision_provisional_video_recommendation_us",
        "planner_live_lag_ms_after",
        "probe_p95_abs_skew_ms",
        "transport/server receive jitter",
        "settling ${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS}s before next segment",
        "print_upstream_calibration_state \"before mirrored run\"",
        "maybe_apply_probe_calibration",
        "calibration_ready=${calibration_ready}",
        "calibration_decision_mode=${calibration_decision_mode}",
        "bounded provisional correction from median skew",
        "bounded provisional correction from analyzer-failure raw activity",
        "raw_failure_calibration_enabled",
        "raw analyzer-failure calibration refused: ",
        "raw_failure_min_pairs",
        "provisional calibration not saved",
        "calibration apply refused: ${calibration_decision_note}",
        "calibrate \"${calibration_apply_audio_delta_us}\" \"${calibration_apply_video_delta_us}\"",
        "LESAVKA_UPSTREAM_BLIND_HEAL is server-side",
        "calibration-save-default",
        "print_upstream_sync_state \"after mirrored run\"",
        "print_upstream_calibration_state \"after mirrored run\"",
        "==> mirrored probe failed",
    ] {
        assert!(
            MIRRORED_SYNC_SCRIPT.contains(expected),
            "mirrored sync script should contain {expected}"
        );
    }
    assert!(
        !MIRRORED_SYNC_SCRIPT.contains("lesavka-sync-probe"),
        "mirrored sync must not use the synthetic direct sender"
    );
}

#[test]
fn local_stimulus_matches_sync_analyzer_pulse_contract() {
    for expected in [
        "--warmup-seconds",
        "--pulse-period-ms",
        "--pulse-width-ms",
        "--marker-tick-period",
        "--audio-gain",
        "/preview",
        "preview_token",
        "observed_preview_token",
        "completed_preview_token",
        "stimulus preview running",
        "stimulus preview completed",
        "audio_state",
        "--event-width-codes",
        "event_width_codes",
        "audio_gain",
        "widthCode",
        "oscillator.frequency.value = 880",
        "setStatus(`ready",
        "Point the real webcam at this window",
    ] {
        assert!(
            LOCAL_STIMULUS.contains(expected),
            "local stimulus should contain {expected}"
        );
    }
}

#[test]
fn manual_probe_python_servers_use_reentrant_state_locks() {
    const BROWSER_CONSUMER: &str = include_str!("../../scripts/manual/browser_consumer_probe.py");
    for (name, script) in [
        ("local stimulus", LOCAL_STIMULUS),
        ("browser consumer", BROWSER_CONSUMER),
    ] {
        assert!(
            script.contains("threading.RLock()"),
            "{name} server request handlers call snapshot while holding state lock; use RLock to avoid /start deadlocks"
        );
    }
}