diff --git a/AGENTS.md b/AGENTS.md index 37036f7..a03555a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -360,3 +360,18 @@ Context: 0.17.15 proved the adaptive/live-edit loop is structurally useful, but - [x] Update manual probe contract tests for continuous browser session behavior. - [x] Run shell syntax checks, focused contract tests, and package checks. - [x] Push clean semver `0.17.16` for installed client/server testing. + +## 0.17.17 Adaptive Segment Failure Continuation Checklist + +Context: 0.17.16 successfully installed and token-verified the first browser capture, but the adaptive run still aborted at segment 1 because `lesavka-sync-analyze` could not form coded pairs (`saw 0`) even though raw activity was nearly synced (`-32.2ms`). That prevented segments 2-4 from exercising the same long-lived Tethys browser session. 0.17.17 keeps adaptive data gathering alive across analyzer-only failures and preserves the failure evidence for summaries. + +- [x] Keep 0.17.17 scoped to probe/tooling reliability; do not change media playout policy. +- [x] Add `BROWSER_ANALYSIS_REQUIRED` so browser captures can keep artifacts even when analyzer exits nonzero. +- [x] In adaptive mirrored mode, treat analyzer-only failures as nonfatal segment evidence. +- [x] Preserve analyzer failure reason, raw activity delta, and log path as structured segment artifacts. +- [x] Continue to abort on browser startup, recording, upload, or capture fetch failures. +- [x] Include analyzer failure artifacts in segment CSV/JSONL summaries. +- [x] Keep calibration apply impossible without a real analyzer `report.json` and `calibration.ready=true`. +- [x] Update manual probe contract tests for analyzer-failure continuation. +- [x] Run shell syntax checks, focused contract tests, and package checks. +- [ ] Push clean semver `0.17.17` for installed client/server testing. diff --git a/Cargo.lock b/Cargo.lock index 03b48c0..9ac5ba7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lesavka_client" -version = "0.17.16" +version = "0.17.17" dependencies = [ "anyhow", "async-stream", @@ -1686,7 +1686,7 @@ dependencies = [ [[package]] name = "lesavka_common" -version = "0.17.16" +version = "0.17.17" dependencies = [ "anyhow", "base64", @@ -1698,7 +1698,7 @@ dependencies = [ [[package]] name = "lesavka_server" -version = "0.17.16" +version = "0.17.17" dependencies = [ "anyhow", "base64", diff --git a/client/Cargo.toml b/client/Cargo.toml index 0fed2ba..c1c8696 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -4,7 +4,7 @@ path = "src/main.rs" [package] name = "lesavka_client" -version = "0.17.16" +version = "0.17.17" edition = "2024" [dependencies] diff --git a/common/Cargo.toml b/common/Cargo.toml index 611b22c..55ca764 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lesavka_common" -version = "0.17.16" +version = "0.17.17" edition = "2024" build = "build.rs" diff --git a/scripts/manual/run_upstream_browser_av_sync.sh b/scripts/manual/run_upstream_browser_av_sync.sh index 95f07f8..6a2e395 100755 --- a/scripts/manual/run_upstream_browser_av_sync.sh +++ b/scripts/manual/run_upstream_browser_av_sync.sh @@ -16,6 +16,7 @@ PROBE_DURATION_SECONDS=${PROBE_DURATION_SECONDS:-15} BROWSER_RECORD_SECONDS=${BROWSER_RECORD_SECONDS:-${PROBE_DURATION_SECONDS}} BROWSER_SYNC_DRIVER_COMMAND=${BROWSER_SYNC_DRIVER_COMMAND:-} BROWSER_CONSUMER_REUSE_SESSION=${BROWSER_CONSUMER_REUSE_SESSION:-0} +BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED:-1} SYNC_ANALYZE_EVENT_WIDTH_CODES=${SYNC_ANALYZE_EVENT_WIDTH_CODES:-} BROWSER_PORT=${BROWSER_PORT:-18443} REMOTE_SCRIPT=${REMOTE_SCRIPT:-/tmp/lesavka-browser-consumer-probe.py} @@ -222,16 +223,70 @@ if ((fetch_status != 0)); then fi echo "==> analyzing browser capture" +mkdir -p "${LOCAL_REPORT_DIR}" analyze_args=(--report-dir "${LOCAL_REPORT_DIR}") if [[ -n "${SYNC_ANALYZE_EVENT_WIDTH_CODES}" ]]; then analyze_args+=(--event-width-codes "${SYNC_ANALYZE_EVENT_WIDTH_CODES}") fi analyze_args+=("${LOCAL_CAPTURE}") +analysis_log="${LOCAL_REPORT_DIR}/analyze.log" +analysis_status=0 ( cd "${REPO_ROOT}" cargo run -p lesavka_client --bin lesavka-sync-analyze -- \ "${analyze_args[@]}" +) 2>&1 | tee "${analysis_log}" || analysis_status=$? +if (( analysis_status != 0 )); then + python3 - "${analysis_log}" "${analysis_status}" "${LOCAL_REPORT_DIR}/analysis-failure.json" <<'PY' +import json +import re +import sys +from pathlib import Path + +log_path = Path(sys.argv[1]) +status = int(sys.argv[2]) +output_path = Path(sys.argv[3]) +text = log_path.read_text(encoding="utf-8", errors="replace") +reason = "" +lines = text.splitlines() +for index, line in enumerate(lines): + if "Caused by:" in line: + for candidate in lines[index + 1:]: + candidate = candidate.strip() + if candidate: + reason = candidate + break +if not reason: + reason = lines[-1].strip() if lines else "analyzer failed" + +raw_match = re.search( + r"raw activity delta was ([+-]?[0-9]+(?:\\.[0-9]+)?) ms " + r"\\(video=([0-9]+(?:\\.[0-9]+)?)s audio=([0-9]+(?:\\.[0-9]+)?)s\\)", + text, ) +paired_match = re.search(r"saw ([0-9]+)", reason) +payload = { + "status": "analysis_error", + "exit_status": status, + "reason": reason, + "analyze_log": str(log_path), + "paired_pulses": int(paired_match.group(1)) if paired_match else 0, +} +if raw_match: + payload.update({ + "raw_activity_delta_ms": float(raw_match.group(1)), + "raw_first_video_activity_s": float(raw_match.group(2)), + "raw_first_audio_activity_s": float(raw_match.group(3)), + }) +output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") +PY + echo "==> analyzer failed for ${LOCAL_CAPTURE}" + echo " ↪ analysis_failure_json=${LOCAL_REPORT_DIR}/analysis-failure.json" + if [[ "${BROWSER_ANALYSIS_REQUIRED}" == "1" ]]; then + exit "${analysis_status}" + fi + echo " ↪ continuing because BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED}" +fi echo "==> done" echo "capture: ${LOCAL_CAPTURE}" diff --git a/scripts/manual/run_upstream_mirrored_av_sync.sh b/scripts/manual/run_upstream_mirrored_av_sync.sh index 695ed82..799083c 100755 --- a/scripts/manual/run_upstream_mirrored_av_sync.sh +++ b/scripts/manual/run_upstream_mirrored_av_sync.sh @@ -30,6 +30,7 @@ LESAVKA_SYNC_SAVE_CALIBRATION=${LESAVKA_SYNC_SAVE_CALIBRATION:-0} LESAVKA_SYNC_CALIBRATION_TARGET=${LESAVKA_SYNC_CALIBRATION_TARGET:-video} LESAVKA_SYNC_CALIBRATION_SEGMENTS=${LESAVKA_SYNC_CALIBRATION_SEGMENTS:-1} LESAVKA_SYNC_CONTINUOUS_BROWSER=${LESAVKA_SYNC_CONTINUOUS_BROWSER:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}} +LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE=${LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}} LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS=${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS:-3} STIMULUS_PORT=${STIMULUS_PORT:-18444} STIMULUS_SETTLE_SECONDS=${STIMULUS_SETTLE_SECONDS:-10} @@ -444,16 +445,22 @@ run_browser_capture_with_real_driver() { local wait_seconds=$((PROBE_DURATION_SECONDS + 2)) local driver_command="curl -fsS -X POST http://127.0.0.1:${STIMULUS_PORT}/start >/dev/null; sleep ${wait_seconds}" local reuse_browser_session=0 + local analysis_required=1 if [[ "${LESAVKA_SYNC_CONTINUOUS_BROWSER}" == "1" && "${segment_index}" != "1" ]]; then reuse_browser_session=1 fi + if [[ "${LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE}" == "1" ]]; then + analysis_required=0 + fi mkdir -p "${segment_output_dir}" echo "==> starting Tethys browser consumer and mirrored driver (${segment_label})" echo " ↪ browser_consumer_reuse_session=${reuse_browser_session}" + echo " ↪ browser_analysis_required=${analysis_required}" BROWSER_RECORD_SECONDS="${record_seconds}" \ PROBE_DURATION_SECONDS="${PROBE_DURATION_SECONDS}" \ BROWSER_SYNC_DRIVER_COMMAND="${driver_command}" \ BROWSER_CONSUMER_REUSE_SESSION="${reuse_browser_session}" \ + BROWSER_ANALYSIS_REQUIRED="${analysis_required}" \ SYNC_ANALYZE_EVENT_WIDTH_CODES="${PROBE_EVENT_WIDTH_CODES}" \ LOCAL_OUTPUT_DIR="${segment_output_dir}" \ LESAVKA_SERVER_ADDR="${RESOLVED_LESAVKA_SERVER_ADDR}" \ @@ -520,6 +527,13 @@ def latest_report(segment_dir): return max(reports, key=lambda path: path.stat().st_mtime) +def latest_analysis_failure(segment_dir): + failures = list(segment_dir.glob("*/analysis-failure.json")) + if not failures: + return None + return max(failures, key=lambda path: path.stat().st_mtime) + + def as_float(value): if value is None or value in {"", "pending"}: return None @@ -551,6 +565,10 @@ for segment in range(1, segment_count + 1): report = json.loads(report_path.read_text(encoding="utf-8")) verdict = report.get("verdict", {}) calibration = report.get("calibration", {}) + failure_path = latest_analysis_failure(segment_dir) + failure = {} + if failure_path is not None: + failure = json.loads(failure_path.read_text(encoding="utf-8")) planner_before = read_env(segment_dir / "planner-before.env") planner_after = read_env(segment_dir / "planner-after.env") @@ -561,15 +579,19 @@ for segment in range(1, segment_count + 1): row = { "segment": segment, "report_json": str(report_path) if report_path else "", - "probe_status": verdict.get("status", "missing"), + "analysis_failure_json": str(failure_path) if failure_path else "", + "analysis_failure_reason": failure.get("reason", ""), + "probe_status": verdict.get("status", failure.get("status", "missing")), "probe_passed": bool(verdict.get("passed", False)), "probe_p95_abs_skew_ms": as_float(str(verdict.get("p95_abs_skew_ms", ""))), "probe_max_abs_skew_ms": as_float(str(verdict.get("max_abs_skew_ms", ""))), "probe_median_skew_ms": as_float(str(report.get("median_skew_ms", ""))), "probe_mean_skew_ms": as_float(str(report.get("mean_skew_ms", ""))), "probe_drift_ms": as_float(str(report.get("drift_ms", ""))), - "probe_paired_pulses": report.get("paired_event_count", 0), - "probe_activity_start_delta_ms": as_float(str(report.get("activity_start_delta_ms", ""))), + "probe_paired_pulses": report.get("paired_event_count", failure.get("paired_pulses", 0)), + "probe_activity_start_delta_ms": as_float(str(report.get("activity_start_delta_ms", failure.get("raw_activity_delta_ms", "")))), + "analysis_raw_first_video_activity_s": as_float(str(failure.get("raw_first_video_activity_s", ""))), + "analysis_raw_first_audio_activity_s": as_float(str(failure.get("raw_first_audio_activity_s", ""))), "calibration_ready": bool(calibration.get("ready", False)), "calibration_note": calibration.get("note", ""), "decision_video_delta_us": as_float(decision.get("calibration_apply_video_delta_us")), diff --git a/server/Cargo.toml b/server/Cargo.toml index 491848f..6a36a67 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -10,7 +10,7 @@ bench = false [package] name = "lesavka_server" -version = "0.17.16" +version = "0.17.17" edition = "2024" autobins = false diff --git a/testing/tests/client_manual_sync_script_contract.rs b/testing/tests/client_manual_sync_script_contract.rs index d7da227..0bf4528 100644 --- a/testing/tests/client_manual_sync_script_contract.rs +++ b/testing/tests/client_manual_sync_script_contract.rs @@ -74,12 +74,15 @@ fn browser_sync_script_can_delegate_to_a_real_path_driver() { "BROWSER_RECORD_SECONDS=${BROWSER_RECORD_SECONDS:-${PROBE_DURATION_SECONDS}}", "BROWSER_SYNC_DRIVER_COMMAND=${BROWSER_SYNC_DRIVER_COMMAND:-}", "BROWSER_CONSUMER_REUSE_SESSION=${BROWSER_CONSUMER_REUSE_SESSION:-0}", + "BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED:-1}", "SYNC_ANALYZE_EVENT_WIDTH_CODES=${SYNC_ANALYZE_EVENT_WIDTH_CODES:-}", "==> running custom browser sync driver", "bash -lc \"${BROWSER_SYNC_DRIVER_COMMAND}\"", "browser_start_token=${browser_start_token}", "uploaded_start_token", "BROWSER_START_TOKEN", + "analysis-failure.json", + "BROWSER_ANALYSIS_REQUIRED=${BROWSER_ANALYSIS_REQUIRED}", "--event-width-codes", "--report-dir \"${LOCAL_REPORT_DIR}\"", "for attempt in 1 2 3 4 5", @@ -121,11 +124,14 @@ fn mirrored_sync_script_uses_real_client_capture_path() { "LESAVKA_SYNC_ADAPTIVE_CALIBRATION=${LESAVKA_SYNC_ADAPTIVE_CALIBRATION:-0}", "LESAVKA_SYNC_CALIBRATION_SEGMENTS=${LESAVKA_SYNC_CALIBRATION_SEGMENTS:-1}", "LESAVKA_SYNC_CONTINUOUS_BROWSER=${LESAVKA_SYNC_CONTINUOUS_BROWSER:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}", + "LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE=${LESAVKA_SYNC_CONTINUE_ON_ANALYSIS_FAILURE:-${LESAVKA_SYNC_ADAPTIVE_CALIBRATION}}", "LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS=${LESAVKA_SYNC_SEGMENT_SETTLE_SECONDS:-3}", "LESAVKA_SYNC_ADAPTIVE_CALIBRATION", "LESAVKA_SYNC_CALIBRATION_SEGMENTS=4", "browser_consumer_reuse_session=${reuse_browser_session}", + "browser_analysis_required=${analysis_required}", "BROWSER_CONSUMER_REUSE_SESSION=\"${reuse_browser_session}\"", + "BROWSER_ANALYSIS_REQUIRED=\"${analysis_required}\"", "LESAVKA_SYNC_CALIBRATION_SEGMENTS must be a positive integer", "run_mirrored_segments", "summarize_adaptive_probe_metrics", @@ -136,6 +142,8 @@ fn mirrored_sync_script_uses_real_client_capture_path() { "calibration-decision.env", "segment-metrics.csv", "segment-metrics.jsonl", + "analysis_failure_reason", + "probe_activity_start_delta_ms", "blind-targets.json", "no segment produced a passing probe verdict; refusing to invent blind targets", "planner_live_lag_ms_after",