test(server-rc): retry signal readiness with layer evidence

Cargo.lock

@@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
 
 [[package]]
 name = "lesavka_client"
-version = "0.19.25"
+version = "0.19.26"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -1686,7 +1686,7 @@ dependencies = [
 
 [[package]]
 name = "lesavka_common"
-version = "0.19.25"
+version = "0.19.26"
 dependencies = [
  "anyhow",
  "base64",
@@ -1698,7 +1698,7 @@ dependencies = [
 
 [[package]]
 name = "lesavka_server"
-version = "0.19.25"
+version = "0.19.26"
 dependencies = [
  "anyhow",
  "base64",

client/Cargo.toml

@@ -4,7 +4,7 @@ path                    = "src/main.rs"
 
 [package]
 name                    = "lesavka_client"
-version                 = "0.19.25"
+version                 = "0.19.26"
 edition                 = "2024"
 
 [dependencies]

common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name                    = "lesavka_common"
-version                 = "0.19.25"
+version                 = "0.19.26"
 edition                 = "2024"
 build                   = "build.rs"
 

scripts/manual/run_server_to_rc_mode_matrix.sh

@@ -78,8 +78,10 @@ LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS=${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_
 LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS:-0}
 LESAVKA_SERVER_RC_SIGNAL_READY=${LESAVKA_SERVER_RC_SIGNAL_READY:-1}
 LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS:-3}
-LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS:-8}
+LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS:-12}
 LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS:-1}
+LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS:-4}
+LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS:-5}
 
 LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS=${LESAVKA_SERVER_RC_MAX_VIDEO_HICCUPS:-0}
 LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS=${LESAVKA_SERVER_RC_MAX_AUDIO_HICCUPS:-0}
@@ -364,6 +366,229 @@ raise SystemExit(0 if passed else 1)
 PY
 }
 
+write_signal_readiness_attempt_result() {
+  local attempt=$1
+  local artifact_dir=$2
+  local run_status=$3
+  local run_log=$4
+  local min_pairs=$5
+  local output_json=$6
+  python3 - <<'PY' \
+    "${attempt}" \
+    "${artifact_dir}" \
+    "${run_status}" \
+    "${run_log}" \
+    "${min_pairs}" \
+    "${output_json}"
+import json
+import pathlib
+import re
+import sys
+
+(
+    attempt_raw,
+    artifact_dir_raw,
+    run_status_raw,
+    run_log_raw,
+    min_pairs_raw,
+    output_json_raw,
+) = sys.argv[1:]
+
+def as_int(value, default=0):
+    try:
+        return int(str(value).strip())
+    except Exception:
+        return default
+
+def as_float(value, default=0.0):
+    try:
+        return float(str(value).strip())
+    except Exception:
+        return default
+
+def load_json(path):
+    try:
+        return json.loads(path.read_text())
+    except Exception:
+        return {}
+
+def read_text(path):
+    try:
+        return path.read_text(errors="replace")
+    except Exception:
+        return ""
+
+attempt = as_int(attempt_raw)
+run_status = as_int(run_status_raw)
+min_pairs = as_int(min_pairs_raw, 1)
+run_log = pathlib.Path(run_log_raw)
+artifact_dir = pathlib.Path(artifact_dir_raw) if artifact_dir_raw else None
+if artifact_dir is None or not artifact_dir.exists():
+    match = re.findall(r"^artifact_dir:\s*(.+)$", read_text(run_log), flags=re.MULTILINE)
+    if match:
+        artifact_dir = pathlib.Path(match[-1].strip())
+if artifact_dir is None:
+    artifact_dir = pathlib.Path("__missing_signal_readiness_artifact__")
+
+report = load_json(artifact_dir / "report.json")
+timeline = load_json(artifact_dir / "server-output-timeline.json")
+error_log = artifact_dir / "analysis-error.log"
+error_text = read_text(error_log)
+if not error_text:
+    error_text = read_text(run_log)
+
+events = timeline.get("events") or []
+server_event_count = len(events)
+server_video_handoff_count = sum(
+    1
+    for event in events
+    if event.get("video_feed_unix_ns") is not None
+    or event.get("video_feed_monotonic_us") is not None
+)
+server_audio_handoff_count = sum(
+    1
+    for event in events
+    if event.get("audio_push_unix_ns") is not None
+    or event.get("audio_push_monotonic_us") is not None
+)
+
+verdict = report.get("verdict") or {}
+signature_coverage = report.get("signature_coverage") or {}
+video_events = as_int(report.get("video_event_count"))
+audio_events = as_int(report.get("audio_event_count"))
+paired_events = as_int(report.get("paired_event_count"))
+coded_pairs = as_int(signature_coverage.get("paired_event_count"), paired_events)
+expected_pairs = as_int(signature_coverage.get("expected_event_count"))
+raw_activity_delta_ms = as_float(report.get("activity_start_delta_ms"))
+analyzer_failure = ""
+
+failure_match = re.search(
+    r"coded pulse common window removed one stream entirely;[^\n]*"
+    r"\(video=(?P<video>\d+)\s+audio=(?P<audio>\d+)\s+raw activity delta\s+"
+    r"(?P<delta>[+-]?\d+(?:\.\d+)?)\s+ms\)",
+    error_text,
+)
+if failure_match:
+    video_events = max(video_events, as_int(failure_match.group("video")))
+    audio_events = max(audio_events, as_int(failure_match.group("audio")))
+    raw_activity_delta_ms = as_float(failure_match.group("delta"))
+    analyzer_failure = failure_match.group(0)
+
+sync_passed = verdict.get("passed") is True
+paired_ready = paired_events >= min_pairs or coded_pairs >= min_pairs
+video_ready = video_events >= min_pairs
+audio_ready = audio_events >= min_pairs
+passed = run_status == 0 and paired_ready and video_ready and audio_ready and sync_passed
+
+reasons = []
+if run_status != 0:
+    reasons.append(f"probe command exited {run_status}")
+if server_event_count == 0:
+    reasons.append("server did not report generated coded events")
+if server_video_handoff_count == 0:
+    reasons.append("server did not report video sink handoff events")
+if server_audio_handoff_count == 0:
+    reasons.append("server did not report audio sink handoff events")
+if video_events < min_pairs:
+    reasons.append(f"Tethys video coded events {video_events} < {min_pairs}")
+if audio_events < min_pairs:
+    reasons.append(f"Tethys audio coded events {audio_events} < {min_pairs}")
+if max(paired_events, coded_pairs) < min_pairs:
+    reasons.append(f"paired coded events {max(paired_events, coded_pairs)} < {min_pairs}")
+if report and not sync_passed:
+    reasons.append(f"sync verdict did not pass: {verdict.get('status', 'unknown')}")
+if analyzer_failure:
+    reasons.append(analyzer_failure)
+if not reasons and passed:
+    reasons.append("ready")
+elif not reasons:
+    reasons.append("signal readiness failed")
+
+result = {
+    "schema": "lesavka.server-rc-signal-readiness-attempt.v1",
+    "attempt": attempt,
+    "passed": passed,
+    "reason": "; ".join(reasons),
+    "run_status": run_status,
+    "run_log": str(run_log),
+    "artifact_dir": str(artifact_dir) if artifact_dir.exists() else "",
+    "analysis_error_log": str(error_log) if error_log.exists() else "",
+    "counts": {
+        "server_events": server_event_count,
+        "server_video_handoffs": server_video_handoff_count,
+        "server_audio_handoffs": server_audio_handoff_count,
+        "video_events": video_events,
+        "audio_events": audio_events,
+        "paired_events": paired_events,
+        "coded_pairs": coded_pairs,
+        "expected_pairs": expected_pairs,
+    },
+    "layers": {
+        "server_generated_video": server_event_count > 0,
+        "server_generated_audio": server_event_count > 0,
+        "server_video_handoff": server_video_handoff_count > 0,
+        "server_audio_handoff": server_audio_handoff_count > 0,
+        "tethys_video_detected": video_events > 0,
+        "tethys_audio_detected": audio_events > 0,
+        "tethys_video_ready": video_ready,
+        "tethys_audio_ready": audio_ready,
+        "paired_coded_ready": paired_ready,
+        "analyzer_sync_passed": sync_passed,
+    },
+    "sync": {
+        "status": verdict.get("status", "unknown"),
+        "passed": sync_passed,
+        "p95_abs_skew_ms": as_float(verdict.get("p95_abs_skew_ms")),
+        "median_skew_ms": as_float(report.get("median_skew_ms")),
+        "drift_ms": as_float(report.get("drift_ms")),
+        "raw_activity_delta_ms": raw_activity_delta_ms,
+    },
+    "analyzer_failure": analyzer_failure,
+}
+pathlib.Path(output_json_raw).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
+print(result["reason"])
+raise SystemExit(0 if passed else 1)
+PY
+}
+
+write_signal_readiness_attempts_summary() {
+  local output_json=$1
+  shift || true
+  python3 - <<'PY' "${output_json}" "$@"
+import json
+import pathlib
+import sys
+
+output = pathlib.Path(sys.argv[1])
+attempts = []
+for raw in sys.argv[2:]:
+    path = pathlib.Path(raw)
+    try:
+        attempts.append(json.loads(path.read_text()))
+    except Exception:
+        attempts.append({
+            "schema": "lesavka.server-rc-signal-readiness-attempt.v1",
+            "attempt_json": str(path),
+            "passed": False,
+            "reason": "attempt result could not be read",
+        })
+passed_attempt = next((attempt for attempt in attempts if attempt.get("passed")), None)
+summary = {
+    "schema": "lesavka.server-rc-signal-readiness-summary.v1",
+    "passed": passed_attempt is not None,
+    "passed_attempt": passed_attempt.get("attempt") if passed_attempt else None,
+    "attempt_count": len(attempts),
+    "final_reason": (
+        "ready"
+        if passed_attempt
+        else (attempts[-1].get("reason") if attempts else "no readiness attempts ran")
+    ),
+    "attempts": attempts,
+}
+output.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
+PY
+}
+
 write_signal_readiness_failure() {
   local mode=$1
   local width=$2
@@ -374,8 +599,9 @@ write_signal_readiness_failure() {
   local readiness_status=$7
   local readiness_log=$8
   local readiness_artifact_dir=$9
-  local readiness_reason=${10}
+  local readiness_attempts_json=${10}
-  local output_json=${11}
+  local readiness_reason=${11}
+  local output_json=${12}
   python3 - <<'PY' \
     "${mode}" \
     "${width}" \
@@ -386,6 +612,7 @@ write_signal_readiness_failure() {
     "${readiness_status}" \
     "${readiness_log}" \
     "${readiness_artifact_dir}" \
+    "${readiness_attempts_json}" \
     "${readiness_reason}" \
     "${output_json}"
 import json
@@ -402,6 +629,7 @@ import sys
     readiness_status_raw,
     readiness_log,
     readiness_artifact_dir,
+    readiness_attempts_json,
     readiness_reason,
     output_json,
 ) = sys.argv[1:]
@@ -421,6 +649,7 @@ def load_json(path):
 report = load_json(pathlib.Path(readiness_artifact_dir) / "report.json")
 verdict = report.get("verdict") or {}
 signature_coverage = report.get("signature_coverage") or {}
+readiness_attempts = load_json(readiness_attempts_json).get("attempts") if readiness_attempts_json else None
 result = {
     "schema": "lesavka.server-rc-mode-result.v1",
     "mode": mode,
@@ -440,6 +669,8 @@ result = {
         "reason": readiness_reason,
         "artifact_dir": readiness_artifact_dir,
         "run_log": readiness_log,
+        "attempts_json": readiness_attempts_json,
+        "attempts": readiness_attempts or [],
     },
     "sync": {
         "passed": verdict.get("passed") is True,
@@ -1516,6 +1747,28 @@ for result in results:
             f"valid={freshness.get('capture_timebase_valid')} "
             f"reason={freshness.get('capture_timebase_reason', '')}"
         )
+    signal_readiness = result.get("signal_readiness") or {}
+    signal_attempts = signal_readiness.get("attempts") or []
+    if signal_readiness:
+        lines.append(
+            "  signal readiness: "
+            f"passed={signal_readiness.get('passed', False)} "
+            f"attempts={len(signal_attempts)} "
+            f"reason={signal_readiness.get('reason', '')}"
+        )
+    for attempt in signal_attempts:
+        counts = attempt.get("counts") or {}
+        layers = attempt.get("layers") or {}
+        lines.append(
+            "  signal attempt "
+            f"{attempt.get('attempt', '?')}: passed={attempt.get('passed', False)} "
+            f"server_events={counts.get('server_events', 0)} "
+            f"video={counts.get('video_events', 0)} audio={counts.get('audio_events', 0)} "
+            f"paired={counts.get('paired_events', counts.get('coded_pairs', 0))} "
+            f"audio_ready={layers.get('tethys_audio_ready', False)} "
+            f"video_ready={layers.get('tethys_video_ready', False)} "
+            f"reason={attempt.get('reason', '')}"
+        )
     if "seed_video_delay_us" in result or "seed_audio_delay_us" in result:
         lines.append(
             "  tuning: "
@@ -1549,7 +1802,7 @@ echo "   ↪ capture_stack=${REMOTE_CAPTURE_STACK} audio_source=${REMOTE_AUDIO_S
 echo "   ↪ tune_delays=${LESAVKA_SERVER_RC_TUNE_DELAYS} confirm=${LESAVKA_SERVER_RC_TUNE_CONFIRM} min_pairs=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS} max_abs_skew_ms=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS} max_step_us=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US} min_change_us=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}"
 echo "   ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS} min_pairs=${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}"
 echo "   ↪ coded_pairs_min=${LESAVKA_SERVER_RC_MIN_CODED_PAIRS} require_all_coded=${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS} smoothness_gate=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS}"
-echo "   ↪ signal_ready=${LESAVKA_SERVER_RC_SIGNAL_READY} min_pairs=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} duration=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}s warmup=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}s"
+echo "   ↪ signal_ready=${LESAVKA_SERVER_RC_SIGNAL_READY} attempts=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS} min_pairs=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} duration=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}s warmup=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}s retry_delay=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s"
 echo "   ↪ reconfigure=${LESAVKA_SERVER_RC_RECONFIGURE} strategy=${LESAVKA_SERVER_RC_RECONFIGURE_STRATEGY} allow_gadget_reset=${LESAVKA_SERVER_RC_ALLOW_GADGET_RESET}"
 echo "   ↪ tethys_ready=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY} settle=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}s timeout=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}s preroll_discard=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}s"
 echo "   ↪ start_delay=${LESAVKA_SERVER_RC_START_DELAY_SECONDS}s"
@@ -1573,6 +1826,7 @@ for mode in "${modes[@]}"; do
   seed_result="${mode_dir}/mode-result-seed.json"
   readiness_dir="${mode_dir}/signal-readiness"
   readiness_log="${readiness_dir}/signal-readiness-run.log"
+  readiness_attempts_json="${readiness_dir}/signal-readiness-attempts.json"
   tuned_log="${mode_dir}/mode-tuned-run.log"
   tuned_result="${mode_dir}/mode-result-tuned.json"
   tune_env="${mode_dir}/mode-tune-candidate.env"
@@ -1585,28 +1839,55 @@ for mode in "${modes[@]}"; do
   if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" ]]; then
     mkdir -p "${readiness_dir}"
     echo "==> mode ${mode}: proving Tethys signal readiness before measured probe"
-    run_mode_probe \
+    readiness_pass=1
-      "${width}" \
+    readiness_status=0
-      "${height}" \
+    readiness_artifact_dir=""
-      "${fps}" \
+    readiness_reason="no readiness attempts ran"
-      "${audio_delay_us}" \
+    readiness_attempt_jsons=()
-      "${video_delay_us}" \
+    for readiness_attempt in $(seq 1 "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}"); do
-      "${readiness_dir}" \
+      attempt_dir="${readiness_dir}/attempt-${readiness_attempt}"
-      "${readiness_log}" \
+      readiness_log="${attempt_dir}/signal-readiness-run.log"
-      "${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}" \
+      readiness_attempt_json="${attempt_dir}/signal-readiness-attempt.json"
-      "${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}" \
+      mkdir -p "${attempt_dir}"
-      "${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}"
+      echo "   ↪ readiness attempt ${readiness_attempt}/${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}: requiring ${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} paired coded events"
-    readiness_status=${RUN_MODE_PROBE_STATUS}
+      run_mode_probe \
-    readiness_artifact_dir="$(artifact_dir_from_log "${readiness_log}" "${readiness_dir}")"
+        "${width}" \
-    readiness_reason="probe command exited ${readiness_status}"
+        "${height}" \
-    if [[ "${readiness_status}" -eq 0 ]]; then
+        "${fps}" \
+        "${audio_delay_us}" \
+        "${video_delay_us}" \
+        "${attempt_dir}" \
+        "${readiness_log}" \
+        "${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}" \
+        "${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}" \
+        "${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}"
+      readiness_status=${RUN_MODE_PROBE_STATUS}
+      readiness_artifact_dir="$(artifact_dir_from_log "${readiness_log}" "${attempt_dir}")"
       set +e
-      readiness_reason="$(signal_readiness_passed "${readiness_artifact_dir}" "${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" 2>&1)"
+      readiness_reason="$(
+        write_signal_readiness_attempt_result \
+          "${readiness_attempt}" \
+          "${readiness_artifact_dir}" \
+          "${readiness_status}" \
+          "${readiness_log}" \
+          "${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \
+          "${readiness_attempt_json}" 2>&1
+      )"
       readiness_pass=$?
       set -e
-    else
+      readiness_attempt_jsons+=("${readiness_attempt_json}")
-      readiness_pass=1
+      if [[ "${readiness_pass}" -eq 0 ]]; then
-    fi
+        echo "   ↪ readiness attempt ${readiness_attempt} passed: ${readiness_reason}"
+        break
+      fi
+      [[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed"
+      echo "   ↪ readiness attempt ${readiness_attempt} failed: ${readiness_reason}"
+      if [[ "${readiness_attempt}" -lt "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}" ]]; then
+        echo "   ↪ waiting ${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s before retrying signal readiness"
+        sleep "${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}"
+      fi
+    done
+    write_signal_readiness_attempts_summary "${readiness_attempts_json}" "${readiness_attempt_jsons[@]}"
     if [[ "${readiness_pass}" -ne 0 ]]; then
       [[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed"
       echo "   ↪ signal readiness failed: ${readiness_reason}"
@@ -1620,6 +1901,7 @@ for mode in "${modes[@]}"; do
         "${readiness_status}" \
         "${readiness_log}" \
         "${readiness_artifact_dir}" \
+        "${readiness_attempts_json}" \
         "${readiness_reason}" \
         "${mode_result}"
       cp "${mode_result}" "${seed_result}"

scripts/manual/run_upstream_av_sync.sh

@@ -90,6 +90,7 @@ REMOTE_CAPTURE=${REMOTE_CAPTURE:-"/tmp/lesavka-output-delay-probe-${STAMP}.mkv"}
 LOCAL_REPORT_DIR="${LOCAL_OUTPUT_DIR%/}/lesavka-output-delay-probe-${STAMP}"
 LOCAL_CAPTURE="${LOCAL_REPORT_DIR}/capture.mkv"
 LOCAL_ANALYSIS_JSON="${LOCAL_REPORT_DIR}/report.json"
+LOCAL_ANALYSIS_ERROR_LOG="${LOCAL_REPORT_DIR}/analysis-error.log"
 LOCAL_REPORT_TXT="${LOCAL_REPORT_DIR}/report.txt"
 LOCAL_EVENTS_CSV="${LOCAL_REPORT_DIR}/events.csv"
 LOCAL_SERVER_PROBE_REPLY="${LOCAL_REPORT_DIR}/server-output-probe-reply.txt"
@@ -532,9 +533,10 @@ retry_remote_artifact_command() {
 
 run_remote_sync_analysis_once() {
   local output_path=$1
+  local error_path=${2:-/dev/null}
   ssh ${SSH_OPTS} "${TETHYS_HOST}" \
     "chmod +x '${REMOTE_ANALYZE_BIN}' && '${REMOTE_ANALYZE_BIN}' '${remote_fetch_capture}' --json --event-width-codes '${PROBE_EVENT_WIDTH_CODES}' ${analysis_window_arg}" \
-    > "${output_path}"
+    > "${output_path}" 2> "${error_path}"
 }
 
 resolve_server_addr() {
@@ -3350,16 +3352,28 @@ REMOTE_NORMALIZE_SCRIPT
     fi
     echo "==> analyzing capture on ${TETHYS_HOST}"
     analysis_tmp="${LOCAL_ANALYSIS_JSON}.tmp"
+    analysis_error_tmp="${LOCAL_ANALYSIS_ERROR_LOG}.tmp"
+    rm -f "${analysis_tmp}" "${analysis_error_tmp}" "${LOCAL_ANALYSIS_ERROR_LOG}"
     set +e
     retry_remote_artifact_command \
       "running remote sync analysis on ${TETHYS_HOST}" \
-      run_remote_sync_analysis_once "${analysis_tmp}"
+      run_remote_sync_analysis_once "${analysis_tmp}" "${analysis_error_tmp}"
     analysis_status=$?
     set -e
     if [[ "${analysis_status}" -eq 0 ]]; then
       mv "${analysis_tmp}" "${LOCAL_ANALYSIS_JSON}"
+      if [[ -s "${analysis_error_tmp}" ]]; then
+        mv "${analysis_error_tmp}" "${LOCAL_ANALYSIS_ERROR_LOG}"
+      else
+        rm -f "${analysis_error_tmp}"
+      fi
     else
       rm -f "${analysis_tmp}"
+      if [[ -s "${analysis_error_tmp}" ]]; then
+        mv "${analysis_error_tmp}" "${LOCAL_ANALYSIS_ERROR_LOG}"
+      else
+        rm -f "${analysis_error_tmp}"
+      fi
     fi
   fi
 
@@ -3505,6 +3519,9 @@ fi
 if [[ -f "${LOCAL_ANALYSIS_JSON}" ]]; then
   echo "report_json: ${LOCAL_ANALYSIS_JSON}"
 fi
+if [[ -f "${LOCAL_ANALYSIS_ERROR_LOG}" ]]; then
+  echo "analysis_error_log: ${LOCAL_ANALYSIS_ERROR_LOG}"
+fi
 if [[ -f "${LOCAL_REPORT_TXT}" ]]; then
   echo "report_txt: ${LOCAL_REPORT_TXT}"
 fi

server/Cargo.toml

@@ -10,7 +10,7 @@ bench                   = false
 
 [package]
 name                    = "lesavka_server"
-version                 = "0.19.25"
+version                 = "0.19.26"
 edition                 = "2024"
 autobins                = false
 

testing/tests/client_manual_sync_script_contract.rs

@@ -33,6 +33,7 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
         "LOCAL_OUTPUT_DIR=${LOCAL_OUTPUT_DIR:-/tmp}",
         "LOCAL_REPORT_DIR=\"${LOCAL_OUTPUT_DIR%/}/lesavka-output-delay-probe-${STAMP}\"",
         "LOCAL_ANALYSIS_JSON=\"${LOCAL_REPORT_DIR}/report.json\"",
+        "LOCAL_ANALYSIS_ERROR_LOG=\"${LOCAL_REPORT_DIR}/analysis-error.log\"",
         "LOCAL_EVENTS_CSV=\"${LOCAL_REPORT_DIR}/events.csv\"",
         "LOCAL_SERVER_PROBE_REPLY=\"${LOCAL_REPORT_DIR}/server-output-probe-reply.txt\"",
         "LOCAL_SERVER_TIMELINE_JSON=\"${LOCAL_REPORT_DIR}/server-output-timeline.json\"",
@@ -194,6 +195,8 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
         "fetching capture from ${TETHYS_HOST}",
         "warning: failed to fetch capture artifact; continuing with remote analysis JSON when available",
         "analysis_tmp=\"${LOCAL_ANALYSIS_JSON}.tmp\"",
+        "analysis_error_tmp=\"${LOCAL_ANALYSIS_ERROR_LOG}.tmp\"",
+        "analysis_error_log: ${LOCAL_ANALYSIS_ERROR_LOG}",
         "event_visibility",
         "Coded event visibility",
         "video_only",
@@ -420,9 +423,19 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() {
         "LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS:-0}",
         "LESAVKA_SERVER_RC_SIGNAL_READY=${LESAVKA_SERVER_RC_SIGNAL_READY:-1}",
         "LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS:-3}",
+        "LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS:-12}",
+        "LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS:-4}",
+        "LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS:-5}",
         "signal_readiness_passed",
+        "write_signal_readiness_attempt_result",
+        "write_signal_readiness_attempts_summary",
+        "schema\": \"lesavka.server-rc-signal-readiness-attempt.v1\"",
+        "schema\": \"lesavka.server-rc-signal-readiness-summary.v1\"",
         "proving Tethys signal readiness before measured probe",
+        "readiness attempt ${readiness_attempt}/${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}",
+        "waiting ${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s before retrying signal readiness",
         "signal readiness did not pass",
+        "signal attempt ",
         "smoothness_required",
         "smoothness_warnings",
         "smoothness warning:",