test: continue adaptive sync probe after analysis errors

AGENTS.md

@@ -360,3 +360,18 @@ Context: 0.17.15 proved the adaptive/live-edit loop is structurally useful, but
 - [x] Update manual probe contract tests for continuous browser session behavior.
 - [x] Run shell syntax checks, focused contract tests, and package checks.
 - [x] Push clean semver `0.17.16` for installed client/server testing.
+
+## 0.17.17 Adaptive Segment Failure Continuation Checklist
+
+Context: 0.17.16 successfully installed and token-verified the first browser capture, but the adaptive run still aborted at segment 1 because `lesavka-sync-analyze` could not form coded pairs (`saw 0`) even though raw activity was nearly synced (`-32.2ms`). That prevented segments 2-4 from exercising the same long-lived Tethys browser session. 0.17.17 keeps adaptive data gathering alive across analyzer-only failures and preserves the failure evidence for summaries.
+
+- [x] Keep 0.17.17 scoped to probe/tooling reliability; do not change media playout policy.
+- [x] Add `BROWSER_ANALYSIS_REQUIRED` so browser captures can keep artifacts even when analyzer exits nonzero.
+- [x] In adaptive mirrored mode, treat analyzer-only failures as nonfatal segment evidence.
+- [x] Preserve analyzer failure reason, raw activity delta, and log path as structured segment artifacts.
+- [x] Continue to abort on browser startup, recording, upload, or capture fetch failures.
+- [x] Include analyzer failure artifacts in segment CSV/JSONL summaries.
+- [x] Keep calibration apply impossible without a real analyzer `report.json` and `calibration.ready=true`.
+- [x] Update manual probe contract tests for analyzer-failure continuation.
+- [x] Run shell syntax checks, focused contract tests, and package checks.
+- [ ] Push clean semver `0.17.17` for installed client/server testing.

Cargo.lock

@@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
 
 [[package]]
 name = "lesavka_client"
-version = "0.17.16"
+version = "0.17.17"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -1686,7 +1686,7 @@ dependencies = [
 
 [[package]]
 name = "lesavka_common"
-version = "0.17.16"
+version = "0.17.17"
 dependencies = [
  "anyhow",
  "base64",
@@ -1698,7 +1698,7 @@ dependencies = [
 
 [[package]]
 name = "lesavka_server"
-version = "0.17.16"
+version = "0.17.17"
 dependencies = [
  "anyhow",
  "base64",

client/Cargo.toml

@@ -4,7 +4,7 @@ path                    = "src/main.rs"
 
 [package]
 name                    = "lesavka_client"
-version                 = "0.17.16"
+version                 = "0.17.17"
 edition                 = "2024"
 
 [dependencies]

common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name                    = "lesavka_common"
-version                 = "0.17.16"
+version                 = "0.17.17"
 edition                 = "2024"
 build                   = "build.rs"
 

scripts/manual/run_upstream_browser_av_sync.sh

@@ -16,6 +16,7 @@ PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-15}
 BROWSER_RECORD_SECONDS=${BROWSER_RECORD_SECONDS:-${PROBE_DURATION_SECONDS}}
 BROWSER_SYNC_DRIVER_COMMAND=${BROWSER_SYNC_DRIVER_COMMAND:-}
 BROWSER_CONSUMER_REUSE_SESSION=${BROWSER_CONSUMER_REUSE_SESSION:-0}
+BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED:-1}
 SYNC_ANALYZE_EVENT_WIDTH_CODES=${SYNC_ANALYZE_EVENT_WIDTH_CODES:-}
 BROWSER_PORT=${BROWSER_PORT:-18443}
 REMOTE_SCRIPT=${REMOTE_SCRIPT:-/tmp/lesavka-browser-consumer-probe.py}
@@ -222,16 +223,70 @@ if ((fetch_status != 0)); then
 fi
 
 echo "==> analyzing browser capture"
+mkdir -p "${LOCAL_REPORT_DIR}"
 analyze_args=(--report-dir "${LOCAL_REPORT_DIR}")
 if [[ -n "${SYNC_ANALYZE_EVENT_WIDTH_CODES}" ]]; then
   analyze_args+=(--event-width-codes "${SYNC_ANALYZE_EVENT_WIDTH_CODES}")
 fi
 analyze_args+=("${LOCAL_CAPTURE}")
+analysis_log="${LOCAL_REPORT_DIR}/analyze.log"
+analysis_status=0
 (
   cd "${REPO_ROOT}"
   cargo run -p lesavka_client --bin lesavka-sync-analyze -- \
     "${analyze_args[@]}"
+) 2>&1 | tee "${analysis_log}" || analysis_status=$?
+if (( analysis_status != 0 )); then
+  python3 - "${analysis_log}" "${analysis_status}" "${LOCAL_REPORT_DIR}/analysis-failure.json" <<'PY'
+import json
+import re
+import sys
+from pathlib import Path
+
+log_path = Path(sys.argv[1])
+status = int(sys.argv[2])
+output_path = Path(sys.argv[3])
+text = log_path.read_text(encoding="utf-8", errors="replace")
+reason = ""
+lines = text.splitlines()
+for index, line in enumerate(lines):
+    if "Caused by:" in line:
+        for candidate in lines[index + 1:]:
+            candidate = candidate.strip()
+            if candidate:
+                reason = candidate
+                break
+if not reason:
+    reason = lines[-1].strip() if lines else "analyzer failed"
+
+raw_match = re.search(
+    r"raw activity delta was ([+-]?[0-9]+(?:\\.[0-9]+)?) ms "
+    r"\\(video=([0-9]+(?:\\.[0-9]+)?)s audio=([0-9]+(?:\\.[0-9]+)?)s\\)",
+    text,
 )
+paired_match = re.search(r"saw ([0-9]+)", reason)
+payload = {
+    "status": "analysis_error",
+    "exit_status": status,
+    "reason": reason,
+    "analyze_log": str(log_path),
+    "paired_pulses": int(paired_match.group(1)) if paired_match else 0,
+}
+if raw_match:
+    payload.update({
+        "raw_activity_delta_ms": float(raw_match.group(1)),
+        "raw_first_video_activity_s": float(raw_match.group(2)),
+        "raw_first_audio_activity_s": float(raw_match.group(3)),
+    })
+output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+PY
+  echo "==> analyzer failed for ${LOCAL_CAPTURE}"
+  echo "   ↪ analysis_failure_json=${LOCAL_REPORT_DIR}/analysis-failure.json"
+  if [[ "${BROWSER_ANALYSIS_REQUIRED}" == "1" ]]; then
+    exit "${analysis_status}"
+  fi
+  echo "   ↪ continuing because BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED}"
+fi
 
 echo "==> done"
 echo "capture: ${LOCAL_CAPTURE}"

scripts/manual/run_upstream_mirrored_av_sync.sh

@@ -30,6 +30,7 @@ LESAVKA_SYNC_SAVE_CALIBRATION=${LESAVKA_SYNC_SAVE_CALIBRATION:-0}
 LESAVKA_SYNC_CALIBRATION_TARGET=${LESAVKA_SYNC_CALIBRATION_TARGET:-video}
 LESAVKA_SYNC_CALIBRATION_SEGMENTS=${LESAVKA_SYNC_CALIBRATION_SEGMENTS:-1}
 LESAVKA_SYNC_CONTINUOUS_BROWSER=${LESAVKA_SYNC_CONTINUOUS_BROWSER:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}
+LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE=${LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}
 LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS=${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS:-3}
 STIMULUS_PORT=${STIMULUS_PORT:-18444}
 STIMULUS_SETTLE_SECONDS=${STIMULUS_SETTLE_SECONDS:-10}
@@ -444,16 +445,22 @@ run_browser_capture_with_real_driver() {
   local wait_seconds=$((PROBE_DURATION_SECONDS + 2))
   local driver_command="curl -fsS -X POST http://127.0.0.1:${STIMULUS_PORT}/start >/dev/null; sleep ${wait_seconds}"
   local reuse_browser_session=0
+  local analysis_required=1
   if [[ "${LESAVKA_SYNC_CONTINUOUS_BROWSER}" == "1" && "${segment_index}" != "1" ]]; then
     reuse_browser_session=1
   fi
+  if [[ "${LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE}" == "1" ]]; then
+    analysis_required=0
+  fi
   mkdir -p "${segment_output_dir}"
   echo "==> starting Tethys browser consumer and mirrored driver (${segment_label})"
   echo "   ↪ browser_consumer_reuse_session=${reuse_browser_session}"
+  echo "   ↪ browser_analysis_required=${analysis_required}"
   BROWSER_RECORD_SECONDS="${record_seconds}" \
   PROBE_DURATION_SECONDS="${PROBE_DURATION_SECONDS}" \
   BROWSER_SYNC_DRIVER_COMMAND="${driver_command}" \
   BROWSER_CONSUMER_REUSE_SESSION="${reuse_browser_session}" \
+  BROWSER_ANALYSIS_REQUIRED="${analysis_required}" \
   SYNC_ANALYZE_EVENT_WIDTH_CODES="${PROBE_EVENT_WIDTH_CODES}" \
   LOCAL_OUTPUT_DIR="${segment_output_dir}" \
   LESAVKA_SERVER_ADDR="${RESOLVED_LESAVKA_SERVER_ADDR}" \
@@ -520,6 +527,13 @@ def latest_report(segment_dir):
     return max(reports, key=lambda path: path.stat().st_mtime)
 
 
+def latest_analysis_failure(segment_dir):
+    failures = list(segment_dir.glob("*/analysis-failure.json"))
+    if not failures:
+        return None
+    return max(failures, key=lambda path: path.stat().st_mtime)
+
+
 def as_float(value):
     if value is None or value in {"", "pending"}:
         return None
@@ -551,6 +565,10 @@ for segment in range(1, segment_count + 1):
         report = json.loads(report_path.read_text(encoding="utf-8"))
         verdict = report.get("verdict", {})
         calibration = report.get("calibration", {})
+    failure_path = latest_analysis_failure(segment_dir)
+    failure = {}
+    if failure_path is not None:
+        failure = json.loads(failure_path.read_text(encoding="utf-8"))
 
     planner_before = read_env(segment_dir / "planner-before.env")
     planner_after = read_env(segment_dir / "planner-after.env")
@@ -561,15 +579,19 @@ for segment in range(1, segment_count + 1):
     row = {
         "segment": segment,
         "report_json": str(report_path) if report_path else "",
-        "probe_status": verdict.get("status", "missing"),
+        "analysis_failure_json": str(failure_path) if failure_path else "",
+        "analysis_failure_reason": failure.get("reason", ""),
+        "probe_status": verdict.get("status", failure.get("status", "missing")),
         "probe_passed": bool(verdict.get("passed", False)),
         "probe_p95_abs_skew_ms": as_float(str(verdict.get("p95_abs_skew_ms", ""))),
         "probe_max_abs_skew_ms": as_float(str(verdict.get("max_abs_skew_ms", ""))),
         "probe_median_skew_ms": as_float(str(report.get("median_skew_ms", ""))),
         "probe_mean_skew_ms": as_float(str(report.get("mean_skew_ms", ""))),
         "probe_drift_ms": as_float(str(report.get("drift_ms", ""))),
-        "probe_paired_pulses": report.get("paired_event_count", 0),
+        "probe_paired_pulses": report.get("paired_event_count", failure.get("paired_pulses", 0)),
-        "probe_activity_start_delta_ms": as_float(str(report.get("activity_start_delta_ms", ""))),
+        "probe_activity_start_delta_ms": as_float(str(report.get("activity_start_delta_ms", failure.get("raw_activity_delta_ms", "")))),
+        "analysis_raw_first_video_activity_s": as_float(str(failure.get("raw_first_video_activity_s", ""))),
+        "analysis_raw_first_audio_activity_s": as_float(str(failure.get("raw_first_audio_activity_s", ""))),
         "calibration_ready": bool(calibration.get("ready", False)),
         "calibration_note": calibration.get("note", ""),
         "decision_video_delta_us": as_float(decision.get("calibration_apply_video_delta_us")),

server/Cargo.toml

@@ -10,7 +10,7 @@ bench                   = false
 
 [package]
 name                    = "lesavka_server"
-version                 = "0.17.16"
+version                 = "0.17.17"
 edition                 = "2024"
 autobins                = false
 

testing/tests/client_manual_sync_script_contract.rs

@@ -74,12 +74,15 @@ fn browser_sync_script_can_delegate_to_a_real_path_driver() {
         "BROWSER_RECORD_SECONDS=${BROWSER_RECORD_SECONDS:-${PROBE_DURATION_SECONDS}}",
         "BROWSER_SYNC_DRIVER_COMMAND=${BROWSER_SYNC_DRIVER_COMMAND:-}",
         "BROWSER_CONSUMER_REUSE_SESSION=${BROWSER_CONSUMER_REUSE_SESSION:-0}",
+        "BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED:-1}",
         "SYNC_ANALYZE_EVENT_WIDTH_CODES=${SYNC_ANALYZE_EVENT_WIDTH_CODES:-}",
         "==> running custom browser sync driver",
         "bash -lc \"${BROWSER_SYNC_DRIVER_COMMAND}\"",
         "browser_start_token=${browser_start_token}",
         "uploaded_start_token",
         "BROWSER_START_TOKEN",
+        "analysis-failure.json",
+        "BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED}",
         "--event-width-codes",
         "--report-dir \"${LOCAL_REPORT_DIR}\"",
         "for attempt in 1 2 3 4 5",
@@ -121,11 +124,14 @@ fn mirrored_sync_script_uses_real_client_capture_path() {
         "LESAVKA_SYNC_ADAPTIVE_CALIBRATION=${LESAVKA_SYNC_ADAPTIVE_CALIBRATION:-0}",
         "LESAVKA_SYNC_CALIBRATION_SEGMENTS=${LESAVKA_SYNC_CALIBRATION_SEGMENTS:-1}",
         "LESAVKA_SYNC_CONTINUOUS_BROWSER=${LESAVKA_SYNC_CONTINUOUS_BROWSER:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
+        "LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE=${LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}",
         "LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS=${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS:-3}",
         "LESAVKA_SYNC_ADAPTIVE_CALIBRATION",
         "LESAVKA_SYNC_CALIBRATION_SEGMENTS=4",
         "browser_consumer_reuse_session=${reuse_browser_session}",
+        "browser_analysis_required=${analysis_required}",
         "BROWSER_CONSUMER_REUSE_SESSION=\"${reuse_browser_session}\"",
+        "BROWSER_ANALYSIS_REQUIRED=\"${analysis_required}\"",
         "LESAVKA_SYNC_CALIBRATION_SEGMENTS must be a positive integer",
         "run_mirrored_segments",
         "summarize_adaptive_probe_metrics",
@@ -136,6 +142,8 @@ fn mirrored_sync_script_uses_real_client_capture_path() {
         "calibration-decision.env",
         "segment-metrics.csv",
         "segment-metrics.jsonl",
+        "analysis_failure_reason",
+        "probe_activity_start_delta_ms",
         "blind-targets.json",
         "no segment produced a passing probe verdict; refusing to invent blind targets",
         "planner_live_lag_ms_after",