bstein/lesavka
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

calibration: aggregate repeated RC mode probes

Browse Source
This commit is contained in:
Brad Stein 2026-05-06 00:29:00 -03:00
parent 9c81ab51d3
commit 40287aca33
6 changed files with 652 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Cargo.lock generated
View File

@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]] [[package]]
name = "lesavka_client" name = "lesavka_client"
version = "0.19.27" version = "0.19.28"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-stream", "async-stream",
@ -1686,7 +1686,7 @@ dependencies = [
[[package]] [[package]]
name = "lesavka_common" name = "lesavka_common"
version = "0.19.27" version = "0.19.28"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"base64", "base64",
@ -1698,7 +1698,7 @@ dependencies = [
[[package]] [[package]]
name = "lesavka_server" name = "lesavka_server"
version = "0.19.27" version = "0.19.28"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"base64", "base64",

2
client/Cargo.toml
View File

@ -4,7 +4,7 @@ path = "src/main.rs"
[package] [package]
name = "lesavka_client" name = "lesavka_client"
version = "0.19.27" version = "0.19.28"
edition = "2024" edition = "2024"
[dependencies] [dependencies]

2
common/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "lesavka_common" name = "lesavka_common"
version = "0.19.27" version = "0.19.28"
edition = "2024" edition = "2024"
build = "build.rs" build = "build.rs"

826
scripts/manual/run_server_to_rc_mode_matrix.sh
View File

@ -66,6 +66,14 @@ LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS:
LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS:-80} LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS:-80}
LESAVKA_SERVER_RC_TUNE_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US:-500000} LESAVKA_SERVER_RC_TUNE_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US:-500000}
LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000} LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US:-5000}
LESAVKA_SERVER_RC_REPEAT_COUNT=${LESAVKA_SERVER_RC_REPEAT_COUNT:-1}
LESAVKA_SERVER_RC_VERBOSE_PROBES=${LESAVKA_SERVER_RC_VERBOSE_PROBES:-1}
LESAVKA_SERVER_RC_STATIC_MIN_RUNS=${LESAVKA_SERVER_RC_STATIC_MIN_RUNS:-3}
LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US=${LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US:-30000}
LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS=${LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS:-35}
LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS=${LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS:-20}
LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS=${LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS:-1}
LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS=${LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS:-0}
LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350} LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS:-350}
LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS:-100} LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS:-100}
@ -117,7 +125,13 @@ MATRIX_SUMMARY_CSV="${MATRIX_REPORT_DIR}/mode-matrix-summary.csv"
MATRIX_SUMMARY_TXT="${MATRIX_REPORT_DIR}/mode-matrix-summary.txt" MATRIX_SUMMARY_TXT="${MATRIX_REPORT_DIR}/mode-matrix-summary.txt"
MATRIX_DELAY_JSON="${MATRIX_REPORT_DIR}/mode-delay-recommendations.json" MATRIX_DELAY_JSON="${MATRIX_REPORT_DIR}/mode-delay-recommendations.json"
MATRIX_DELAY_ENV="${MATRIX_REPORT_DIR}/mode-delay-recommendations.env" MATRIX_DELAY_ENV="${MATRIX_REPORT_DIR}/mode-delay-recommendations.env"
MATRIX_STATIC_JSON="${MATRIX_REPORT_DIR}/mode-static-calibration.json"
MATRIX_STATIC_CSV="${MATRIX_REPORT_DIR}/mode-static-calibration.csv"
MATRIX_STATIC_TXT="${MATRIX_REPORT_DIR}/mode-static-calibration.txt"
MATRIX_STATIC_ENV="${MATRIX_REPORT_DIR}/mode-static-calibration.env"
MATRIX_RUN_LOG="${MATRIX_REPORT_DIR}/mode-matrix-run.log"
mkdir -p "${MATRIX_REPORT_DIR}" mkdir -p "${MATRIX_REPORT_DIR}"
exec > >(tee -a "${MATRIX_RUN_LOG}") 2>&1
mode_id() { mode_id() {
local mode=$1 local mode=$1
@ -194,52 +208,60 @@ run_mode_probe() {
fi fi
fi fi
local -a probe_env=(
"TETHYS_HOST=${TETHYS_HOST}"
"LESAVKA_SERVER_HOST=${LESAVKA_SERVER_HOST}"
"LESAVKA_SERVER_CONNECT_HOST=${LESAVKA_SERVER_CONNECT_HOST}"
"LESAVKA_SERVER_ADDR=${LESAVKA_SERVER_ADDR}"
"LESAVKA_SERVER_SCHEME=${LESAVKA_SERVER_SCHEME}"
"LESAVKA_TLS_DOMAIN=${LESAVKA_TLS_DOMAIN}"
"SSH_OPTS=${SSH_OPTS}"
"REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL}"
"REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE}"
"REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK}"
"REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE}"
"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}"
"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}"
"REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS}"
"PROBE_PREBUILD=0"
"VIDEO_SIZE=${width}x${height}"
"VIDEO_FPS=${fps}"
"VIDEO_FORMAT=mjpeg"
"REMOTE_EXPECT_UVC_WIDTH=${width}"
"REMOTE_EXPECT_UVC_HEIGHT=${height}"
"REMOTE_EXPECT_UVC_FPS=${fps}"
"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${audio_delay_us}"
"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${video_delay_us}"
"LESAVKA_OUTPUT_DELAY_APPLY=0"
"LESAVKA_OUTPUT_DELAY_SAVE=0"
"LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0"
"LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${min_pairs}"
"LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}"
"LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}"
"LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}"
"LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}"
"LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}"
"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}"
"LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=${min_pairs}"
"PROBE_EVENT_WIDTH_CODES=${PROBE_EVENT_WIDTH_CODES}"
"PROBE_CONDITIONING_SECONDS=${conditioning_seconds}"
"PROBE_CONDITIONING_WARMUP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS}"
"PROBE_CONDITIONING_GAP_SECONDS=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS}"
"PROBE_CONDITIONING_EVENT_WIDTH_CODES=${LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES}"
"PROBE_DURATION_SECONDS=${probe_duration_seconds}"
"PROBE_WARMUP_SECONDS=${probe_warmup_seconds}"
"ANALYSIS_TIMELINE_WINDOW=${analysis_timeline_window}"
"LOCAL_OUTPUT_DIR=${output_dir}"
)
set +e set +e
TETHYS_HOST="${TETHYS_HOST}" \ if [[ "${LESAVKA_SERVER_RC_VERBOSE_PROBES}" == "0" ]]; then
LESAVKA_SERVER_HOST="${LESAVKA_SERVER_HOST}" \ env "${probe_env[@]}" "${SCRIPT_DIR}/run_upstream_av_sync.sh" >"${log_path}" 2>&1
LESAVKA_SERVER_CONNECT_HOST="${LESAVKA_SERVER_CONNECT_HOST}" \ RUN_MODE_PROBE_STATUS=$?
LESAVKA_SERVER_ADDR="${LESAVKA_SERVER_ADDR}" \ else
LESAVKA_SERVER_SCHEME="${LESAVKA_SERVER_SCHEME}" \ env "${probe_env[@]}" "${SCRIPT_DIR}/run_upstream_av_sync.sh" 2>&1 | tee "${log_path}"
LESAVKA_TLS_DOMAIN="${LESAVKA_TLS_DOMAIN}" \ RUN_MODE_PROBE_STATUS=${PIPESTATUS[0]}
SSH_OPTS="${SSH_OPTS}" \ fi
REMOTE_PULSE_CAPTURE_TOOL="${REMOTE_PULSE_CAPTURE_TOOL}" \
REMOTE_PULSE_VIDEO_MODE="${REMOTE_PULSE_VIDEO_MODE}" \
REMOTE_CAPTURE_STACK="${REMOTE_CAPTURE_STACK}" \
REMOTE_AUDIO_SOURCE="${REMOTE_AUDIO_SOURCE}" \
REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK="${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" \
REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS="${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}" \
REMOTE_CAPTURE_READY_SETTLE_SECONDS="${REMOTE_CAPTURE_READY_SETTLE_SECONDS}" \
PROBE_PREBUILD=0 \
VIDEO_SIZE="${width}x${height}" \
VIDEO_FPS="${fps}" \
VIDEO_FORMAT=mjpeg \
REMOTE_EXPECT_UVC_WIDTH="${width}" \
REMOTE_EXPECT_UVC_HEIGHT="${height}" \
REMOTE_EXPECT_UVC_FPS="${fps}" \
LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US="${audio_delay_us}" \
LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US="${video_delay_us}" \
LESAVKA_OUTPUT_DELAY_APPLY=0 \
LESAVKA_OUTPUT_DELAY_SAVE=0 \
LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=0 \
LESAVKA_OUTPUT_DELAY_MIN_PAIRS="${min_pairs}" \
LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS="${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS}" \
LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_TUNE_MAX_DRIFT_MS}" \
LESAVKA_OUTPUT_DELAY_MAX_STEP_US="${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US}" \
LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}" \
LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_DRIFT_MS}" \
LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS="${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}" \
LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS="${min_pairs}" \
PROBE_EVENT_WIDTH_CODES="${PROBE_EVENT_WIDTH_CODES}" \
PROBE_CONDITIONING_SECONDS="${conditioning_seconds}" \
PROBE_CONDITIONING_WARMUP_SECONDS="${LESAVKA_SERVER_RC_SIGNAL_CONDITION_WARMUP_SECONDS}" \
PROBE_CONDITIONING_GAP_SECONDS="${LESAVKA_SERVER_RC_SIGNAL_CONDITION_GAP_SECONDS}" \
PROBE_CONDITIONING_EVENT_WIDTH_CODES="${LESAVKA_SERVER_RC_SIGNAL_CONDITION_EVENT_WIDTH_CODES}" \
PROBE_DURATION_SECONDS="${probe_duration_seconds}" \
PROBE_WARMUP_SECONDS="${probe_warmup_seconds}" \
ANALYSIS_TIMELINE_WINDOW="${analysis_timeline_window}" \
LOCAL_OUTPUT_DIR="${output_dir}" \
"${SCRIPT_DIR}/run_upstream_av_sync.sh" 2>&1 | tee "${log_path}"
RUN_MODE_PROBE_STATUS=${PIPESTATUS[0]}
set -e set -e
} }
@ -1552,8 +1574,63 @@ pathlib.Path(output_json).write_text(json.dumps(artifact, indent=2, sort_keys=Tr
PY PY
} }
annotate_mode_result() {
local output_json=$1
local mode_id=$2
local mode_run_index=$3
local repeat_index=$4
local repeat_count=$5
local matrix_sequence=$6
local phase=$7
python3 - <<'PY' \
"${output_json}" \
"${mode_id}" \
"${mode_run_index}" \
"${repeat_index}" \
"${repeat_count}" \
"${matrix_sequence}" \
"${phase}"
import json
import pathlib
import sys
(
output_json_raw,
mode_id,
mode_run_index_raw,
repeat_index_raw,
repeat_count_raw,
matrix_sequence_raw,
phase,
) = sys.argv[1:]
def as_int(value, default=0):
try:
return int(str(value).strip())
except Exception:
return default
path = pathlib.Path(output_json_raw)
result = json.loads(path.read_text())
result.update(
{
"mode_id": mode_id,
"mode_run_index": as_int(mode_run_index_raw),
"repeat_index": as_int(repeat_index_raw),
"repeat_count": as_int(repeat_count_raw),
"matrix_sequence": as_int(matrix_sequence_raw),
"probe_phase": phase,
}
)
path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
PY
}
summarize_matrix() { summarize_matrix() {
python3 - <<'PY' "${MATRIX_REPORT_DIR}" "${MATRIX_SUMMARY_JSON}" "${MATRIX_SUMMARY_CSV}" "${MATRIX_SUMMARY_TXT}" "${MATRIX_DELAY_JSON}" "${MATRIX_DELAY_ENV}" python3 - <<'PY' "${MATRIX_REPORT_DIR}" "${MATRIX_SUMMARY_JSON}" "${MATRIX_SUMMARY_CSV}" "${MATRIX_SUMMARY_TXT}" "${MATRIX_DELAY_JSON}" "${MATRIX_DELAY_ENV}" "${MATRIX_STATIC_JSON}" "${MATRIX_STATIC_CSV}" "${MATRIX_STATIC_TXT}" "${MATRIX_STATIC_ENV}" "${LESAVKA_SERVER_RC_STATIC_MIN_RUNS}" "${LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US}" "${LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS}" "${LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS}" "${LESAVKA_SERVER_RC_STATIC_REQUIRE_FRESHNESS}" "${LESAVKA_SERVER_RC_STATIC_REQUIRE_SMOOTHNESS}"
import csv import csv
import json import json
import pathlib import pathlib
@ -1566,6 +1643,49 @@ summary_csv = pathlib.Path(sys.argv[3])
summary_txt = pathlib.Path(sys.argv[4]) summary_txt = pathlib.Path(sys.argv[4])
delay_json = pathlib.Path(sys.argv[5]) delay_json = pathlib.Path(sys.argv[5])
delay_env = pathlib.Path(sys.argv[6]) delay_env = pathlib.Path(sys.argv[6])
static_json = pathlib.Path(sys.argv[7])
static_csv = pathlib.Path(sys.argv[8])
static_txt = pathlib.Path(sys.argv[9])
static_env = pathlib.Path(sys.argv[10])
static_min_runs = int(sys.argv[11])
static_max_spread_us = int(sys.argv[12])
static_max_p95_skew_ms = float(sys.argv[13])
static_max_median_skew_ms = float(sys.argv[14])
static_require_freshness = sys.argv[15].strip().lower() not in {"", "0", "false", "no", "off"}
static_require_smoothness = sys.argv[16].strip().lower() not in {"", "0", "false", "no", "off"}
def as_int(value, default=0):
try:
return int(str(value).strip())
except Exception:
return default
def as_float(value, default=0.0):
try:
return float(str(value).strip())
except Exception:
return default
def median(values, default=None):
cleaned = sorted(value for value in values if value is not None)
if not cleaned:
return default
mid = len(cleaned) // 2
if len(cleaned) % 2:
return cleaned[mid]
return (cleaned[mid - 1] + cleaned[mid]) / 2.0
def median_int(values, default=None):
value = median(values, default)
if value is None:
return None
return int(round(value))
results = [] results = []
for path in sorted(root.glob("*/mode-result.json")): for path in sorted(root.glob("*/mode-result.json")):
try: try:
@ -1582,57 +1702,192 @@ for path in sorted(root.glob("*/mode-result.json")):
pass pass
if tuned_path.exists(): if tuned_path.exists():
result["tuned_result_json"] = str(tuned_path) result["tuned_result_json"] = str(tuned_path)
result["result_json"] = str(path)
results.append(result) results.append(result)
except Exception: except Exception:
continue continue
results.sort(
key=lambda item: (
item.get("mode") or "",
as_int(item.get("mode_run_index"), as_int(item.get("matrix_sequence"), 0)),
as_int(item.get("matrix_sequence"), 0),
)
)
delay_recommendations = {} mode_groups = {}
video_delay_entries = []
audio_delay_entries = []
for result in results: for result in results:
mode = result.get("mode") mode = result.get("mode")
if not mode: if not mode:
continue continue
mode_groups.setdefault(mode, []).append(result)
def static_eligibility(result):
sync = result.get("sync") or {} sync = result.get("sync") or {}
freshness = result.get("freshness") or {}
calibration = result.get("output_delay_calibration") or {} calibration = result.get("output_delay_calibration") or {}
required_pairs = calibration.get("min_pairs") or 13 reasons = []
confirmed = ( required_pairs = as_int(calibration.get("min_pairs"), 13) or 13
sync.get("passed") is True if result.get("passed") is not True:
and (sync.get("paired_event_count") or 0) >= required_pairs reasons.append("probe did not pass all required gates")
) if sync.get("passed") is not True:
candidate_video = calibration.get("video_target_offset_us") reasons.append(f"sync did not pass ({sync.get('status', 'unknown')})")
candidate_audio = calibration.get("audio_target_offset_us") if as_int(sync.get("paired_event_count"), 0) < required_pairs:
video_delay = result.get("video_delay_us") reasons.append(f"paired events {sync.get('paired_event_count', 0)} < {required_pairs}")
audio_delay = result.get("audio_delay_us") if as_float(sync.get("p95_abs_skew_ms"), 0.0) > static_max_p95_skew_ms:
status = "confirmed" if confirmed else "tested" reasons.append(
candidate_available = ( f"p95 skew {as_float(sync.get('p95_abs_skew_ms'), 0.0):.1f}ms > {static_max_p95_skew_ms:.1f}ms"
calibration.get("ready") is True )
and (calibration.get("paired_event_count") or 0) > 0 if abs(as_float(sync.get("median_skew_ms"), 0.0)) > static_max_median_skew_ms:
and isinstance(candidate_video, int) reasons.append(
and isinstance(candidate_audio, int) f"median skew {as_float(sync.get('median_skew_ms'), 0.0):+.1f}ms outside +/-{static_max_median_skew_ms:.1f}ms"
) )
if not confirmed and candidate_available: if static_require_freshness and freshness.get("status") != "pass":
video_delay = candidate_video reasons.append(f"freshness did not pass ({freshness.get('status', 'unknown')})")
audio_delay = candidate_audio if static_require_smoothness and result.get("smoothness_warnings"):
status = "candidate_unconfirmed" reasons.append("smoothness warnings present")
elif not confirmed: if not isinstance(result.get("video_delay_us"), int) or not isinstance(result.get("audio_delay_us"), int):
status = "unavailable" reasons.append("tested delay values unavailable")
video_delay = None return reasons
audio_delay = None
if status != "unavailable" and isinstance(video_delay, int) and isinstance(audio_delay, int):
video_delay_entries.append(f"{mode}={video_delay}") static_modes = {}
audio_delay_entries.append(f"{mode}={audio_delay}") delay_recommendations = {}
delay_recommendations[mode] = { video_delay_entries = []
audio_delay_entries = []
static_video_entries = []
static_audio_entries = []
for mode, mode_results in sorted(mode_groups.items()):
eligible = []
rejected = []
for result in mode_results:
reasons = static_eligibility(result)
if reasons:
rejected.append(
{
"mode_run_index": result.get("mode_run_index"),
"artifact_dir": result.get("artifact_dir"),
"reasons": reasons,
"tested_video_delay_us": result.get("video_delay_us"),
"tested_audio_delay_us": result.get("audio_delay_us"),
"p95_abs_skew_ms": (result.get("sync") or {}).get("p95_abs_skew_ms"),
"median_skew_ms": (result.get("sync") or {}).get("median_skew_ms"),
}
)
else:
eligible.append(result)
tested_video = [result.get("video_delay_us") for result in eligible]
tested_audio = [result.get("audio_delay_us") for result in eligible]
target_video = [
(result.get("output_delay_calibration") or {}).get("video_target_offset_us")
for result in eligible
if isinstance((result.get("output_delay_calibration") or {}).get("video_target_offset_us"), int)
]
target_audio = [
(result.get("output_delay_calibration") or {}).get("audio_target_offset_us")
for result in eligible
if isinstance((result.get("output_delay_calibration") or {}).get("audio_target_offset_us"), int)
]
tested_video_spread = (max(tested_video) - min(tested_video)) if tested_video else None
tested_audio_spread = (max(tested_audio) - min(tested_audio)) if tested_audio else None
target_video_spread = (max(target_video) - min(target_video)) if target_video else None
recommended_video = median_int(tested_video)
recommended_audio = median_int(tested_audio)
status_reasons = []
if len(eligible) < static_min_runs:
status_reasons.append(f"eligible runs {len(eligible)} < {static_min_runs}")
if tested_video_spread is not None and tested_video_spread > static_max_spread_us:
status_reasons.append(f"tested video delay spread {tested_video_spread}us > {static_max_spread_us}us")
if tested_audio_spread is not None and tested_audio_spread > static_max_spread_us:
status_reasons.append(f"tested audio delay spread {tested_audio_spread}us > {static_max_spread_us}us")
if recommended_video is None or recommended_audio is None:
status_reasons.append("no recommended delay could be derived")
if len(eligible) < static_min_runs:
status = "needs_more_runs"
elif status_reasons:
status = "unstable"
else:
status = "ready"
p95_values = [as_float((result.get("sync") or {}).get("p95_abs_skew_ms"), 0.0) for result in eligible]
median_skews = [as_float((result.get("sync") or {}).get("median_skew_ms"), 0.0) for result in eligible]
freshness_budgets = [
as_float((result.get("freshness") or {}).get("worst_event_age_with_uncertainty_ms"), 0.0)
for result in eligible
]
static_entry = {
"status": status, "status": status,
"audio_delay_us": audio_delay, "ready": status == "ready",
"video_delay_us": video_delay, "reasons": status_reasons,
"tested_audio_delay_us": result.get("audio_delay_us"), "mode": mode,
"tested_video_delay_us": result.get("video_delay_us"), "total_runs": len(mode_results),
"sync_status": sync.get("status"), "eligible_runs": len(eligible),
"median_skew_ms": sync.get("median_skew_ms"), "rejected_runs": rejected,
"paired_event_count": sync.get("paired_event_count"), "recommended_video_delay_us": recommended_video,
"paired_confidence_median": sync.get("paired_confidence_median"), "recommended_audio_delay_us": recommended_audio,
"activity_pair_disagreement_ms": sync.get("activity_pair_disagreement_ms"), "tested_video_delay_us": tested_video,
"tested_audio_delay_us": tested_audio,
"tested_video_delay_min_us": min(tested_video) if tested_video else None,
"tested_video_delay_max_us": max(tested_video) if tested_video else None,
"tested_video_delay_spread_us": tested_video_spread,
"tested_audio_delay_spread_us": tested_audio_spread,
"target_video_delay_us": target_video,
"target_audio_delay_us": target_audio,
"target_video_delay_median_us": median_int(target_video),
"target_audio_delay_median_us": median_int(target_audio),
"target_video_delay_spread_us": target_video_spread,
"sync_p95_abs_skew_ms_max": max(p95_values) if p95_values else None,
"sync_p95_abs_skew_ms_median": median(p95_values),
"sync_abs_median_skew_ms_max": max((abs(value) for value in median_skews), default=None),
"freshness_budget_ms_max": max(freshness_budgets) if freshness_budgets else None,
"eligible_artifact_dirs": [result.get("artifact_dir") for result in eligible],
"eligible_result_json": [result.get("result_json") for result in eligible],
}
static_modes[mode] = static_entry
if status == "ready" and isinstance(recommended_video, int) and isinstance(recommended_audio, int):
static_video_entries.append(f"{mode}={recommended_video}")
static_audio_entries.append(f"{mode}={recommended_audio}")
delay_video = recommended_video
delay_audio = recommended_audio
recommendation_status = "static_ready" if status == "ready" else status
if status != "ready":
fallback = (eligible or mode_results)[-1]
fallback_calibration = fallback.get("output_delay_calibration") or {}
fallback_video_target = fallback_calibration.get("video_target_offset_us")
fallback_audio_target = fallback_calibration.get("audio_target_offset_us")
if not isinstance(delay_video, int) or not isinstance(delay_audio, int):
if (
fallback_calibration.get("ready") is True
and isinstance(fallback_video_target, int)
and isinstance(fallback_audio_target, int)
):
delay_video = fallback_video_target
delay_audio = fallback_audio_target
recommendation_status = "candidate_unconfirmed"
else:
delay_video = fallback.get("video_delay_us")
delay_audio = fallback.get("audio_delay_us")
recommendation_status = "tested_unstable"
else:
recommendation_status = f"candidate_{status}"
if isinstance(delay_video, int) and isinstance(delay_audio, int):
video_delay_entries.append(f"{mode}={delay_video}")
audio_delay_entries.append(f"{mode}={delay_audio}")
delay_recommendations[mode] = {
"status": recommendation_status,
"static_status": status,
"audio_delay_us": delay_audio,
"video_delay_us": delay_video,
"eligible_runs": len(eligible),
"total_runs": len(mode_results),
"tested_video_delay_spread_us": tested_video_spread,
"sync_p95_abs_skew_ms_max": static_entry["sync_p95_abs_skew_ms_max"],
"sync_abs_median_skew_ms_max": static_entry["sync_abs_median_skew_ms_max"],
"freshness_budget_ms_max": static_entry["freshness_budget_ms_max"],
"reasons": status_reasons,
} }
summary = { summary = {
@ -1642,6 +1897,12 @@ summary = {
"mode_count": len(results), "mode_count": len(results),
"delay_recommendations_json": str(delay_json), "delay_recommendations_json": str(delay_json),
"delay_recommendations_env": str(delay_env), "delay_recommendations_env": str(delay_env),
"static_calibration_json": str(static_json),
"static_calibration_csv": str(static_csv),
"static_calibration_txt": str(static_txt),
"static_calibration_env": str(static_env),
"static_ready": bool(static_modes) and all(entry.get("ready") for entry in static_modes.values()),
"static_calibration": static_modes,
"delay_recommendations": delay_recommendations, "delay_recommendations": delay_recommendations,
"results": results, "results": results,
} }
@ -1666,6 +1927,37 @@ delay_json.write_text(
) )
+ "\n" + "\n"
) )
static_json.write_text(
json.dumps(
{
"schema": "lesavka.server-rc-static-calibration.v1",
"artifact_dir": str(root),
"ready": summary["static_ready"],
"criteria": {
"min_runs": static_min_runs,
"max_spread_us": static_max_spread_us,
"max_p95_skew_ms": static_max_p95_skew_ms,
"max_median_skew_ms": static_max_median_skew_ms,
"require_freshness": static_require_freshness,
"require_smoothness": static_require_smoothness,
},
"video_delays_us": {
mode: entry.get("recommended_video_delay_us")
for mode, entry in static_modes.items()
if entry.get("ready")
},
"audio_delays_us": {
mode: entry.get("recommended_audio_delay_us")
for mode, entry in static_modes.items()
if entry.get("ready")
},
"modes": static_modes,
},
indent=2,
sort_keys=True,
)
+ "\n"
)
delay_env.write_text( delay_env.write_text(
"LESAVKA_SERVER_RC_MODE_DELAYS_US=" "LESAVKA_SERVER_RC_MODE_DELAYS_US="
+ shlex.quote(",".join(video_delay_entries)) + shlex.quote(",".join(video_delay_entries))
@ -1674,9 +1966,20 @@ delay_env.write_text(
+ shlex.quote(",".join(audio_delay_entries)) + shlex.quote(",".join(audio_delay_entries))
+ "\n" + "\n"
) )
static_env.write_text(
"LESAVKA_SERVER_RC_MODE_DELAYS_US="
+ shlex.quote(",".join(static_video_entries))
+ "\n"
+ "LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US="
+ shlex.quote(",".join(static_audio_entries))
+ "\n"
)
fieldnames = [ fieldnames = [
"mode", "mode",
"mode_run_index",
"repeat_index",
"matrix_sequence",
"passed", "passed",
"seed_video_delay_us", "seed_video_delay_us",
"seed_audio_delay_us", "seed_audio_delay_us",
@ -1702,6 +2005,7 @@ fieldnames = [
"calibration_video_target_offset_us", "calibration_video_target_offset_us",
"calibration_audio_target_offset_us", "calibration_audio_target_offset_us",
"artifact_dir", "artifact_dir",
"result_json",
] ]
with summary_csv.open("w", newline="", encoding="utf-8") as handle: with summary_csv.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames) writer = csv.DictWriter(handle, fieldnames=fieldnames)
@ -1709,6 +2013,9 @@ with summary_csv.open("w", newline="", encoding="utf-8") as handle:
for result in results: for result in results:
writer.writerow({ writer.writerow({
"mode": result.get("mode"), "mode": result.get("mode"),
"mode_run_index": result.get("mode_run_index"),
"repeat_index": result.get("repeat_index"),
"matrix_sequence": result.get("matrix_sequence"),
"passed": result.get("passed"), "passed": result.get("passed"),
"seed_video_delay_us": result.get("seed_video_delay_us"), "seed_video_delay_us": result.get("seed_video_delay_us"),
"seed_audio_delay_us": result.get("seed_audio_delay_us"), "seed_audio_delay_us": result.get("seed_audio_delay_us"),
@ -1734,12 +2041,60 @@ with summary_csv.open("w", newline="", encoding="utf-8") as handle:
"calibration_video_target_offset_us": (result.get("output_delay_calibration") or {}).get("video_target_offset_us"), "calibration_video_target_offset_us": (result.get("output_delay_calibration") or {}).get("video_target_offset_us"),
"calibration_audio_target_offset_us": (result.get("output_delay_calibration") or {}).get("audio_target_offset_us"), "calibration_audio_target_offset_us": (result.get("output_delay_calibration") or {}).get("audio_target_offset_us"),
"artifact_dir": result.get("artifact_dir"), "artifact_dir": result.get("artifact_dir"),
"result_json": result.get("result_json"),
}) })
static_fieldnames = [
"mode",
"status",
"ready",
"eligible_runs",
"total_runs",
"recommended_video_delay_us",
"recommended_audio_delay_us",
"tested_video_delay_min_us",
"tested_video_delay_max_us",
"tested_video_delay_spread_us",
"target_video_delay_median_us",
"target_video_delay_spread_us",
"sync_p95_abs_skew_ms_max",
"sync_p95_abs_skew_ms_median",
"sync_abs_median_skew_ms_max",
"freshness_budget_ms_max",
"reasons",
]
with static_csv.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=static_fieldnames)
writer.writeheader()
for mode, entry in sorted(static_modes.items()):
writer.writerow(
{
"mode": mode,
"status": entry.get("status"),
"ready": entry.get("ready"),
"eligible_runs": entry.get("eligible_runs"),
"total_runs": entry.get("total_runs"),
"recommended_video_delay_us": entry.get("recommended_video_delay_us"),
"recommended_audio_delay_us": entry.get("recommended_audio_delay_us"),
"tested_video_delay_min_us": entry.get("tested_video_delay_min_us"),
"tested_video_delay_max_us": entry.get("tested_video_delay_max_us"),
"tested_video_delay_spread_us": entry.get("tested_video_delay_spread_us"),
"target_video_delay_median_us": entry.get("target_video_delay_median_us"),
"target_video_delay_spread_us": entry.get("target_video_delay_spread_us"),
"sync_p95_abs_skew_ms_max": entry.get("sync_p95_abs_skew_ms_max"),
"sync_p95_abs_skew_ms_median": entry.get("sync_p95_abs_skew_ms_median"),
"sync_abs_median_skew_ms_max": entry.get("sync_abs_median_skew_ms_max"),
"freshness_budget_ms_max": entry.get("freshness_budget_ms_max"),
"reasons": "; ".join(entry.get("reasons") or []),
}
)
lines = [ lines = [
f"Server-to-RC mode matrix for {root}", f"Server-to-RC mode matrix for {root}",
f"- modes: {len(results)}", f"- runs: {len(results)}",
f"- unique modes: {len(mode_groups)}",
f"- verdict: {'pass' if summary['passed'] else 'fail'}", f"- verdict: {'pass' if summary['passed'] else 'fail'}",
f"- static calibration: {'ready' if summary['static_ready'] else 'needs-more-data'}",
] ]
for result in results: for result in results:
sync = result.get("sync") or {} sync = result.get("sync") or {}
@ -1748,7 +2103,7 @@ for result in results:
calibration = result.get("output_delay_calibration") or {} calibration = result.get("output_delay_calibration") or {}
marker = "PASS" if result.get("passed") else "FAIL" marker = "PASS" if result.get("passed") else "FAIL"
lines.append( lines.append(
f"- {marker} {result.get('mode')}: " f"- {marker} {result.get('mode')} run={result.get('mode_run_index', '?')}: "
f"delays video={result.get('video_delay_us', 0)}us audio={result.get('audio_delay_us', 0)}us; " f"delays video={result.get('video_delay_us', 0)}us audio={result.get('audio_delay_us', 0)}us; "
f"sync {sync.get('status')} p95={sync.get('p95_abs_skew_ms', 0.0):.1f}ms median={sync.get('median_skew_ms', 0.0):+.1f}ms; " f"sync {sync.get('status')} p95={sync.get('p95_abs_skew_ms', 0.0):.1f}ms median={sync.get('median_skew_ms', 0.0):+.1f}ms; "
f"freshness {freshness.get('status')} budget={freshness.get('worst_event_age_with_uncertainty_ms') or 0.0:.1f}ms; " f"freshness {freshness.get('status')} budget={freshness.get('worst_event_age_with_uncertainty_ms') or 0.0:.1f}ms; "
@ -1824,7 +2179,41 @@ for result in results:
if video_delay_entries or audio_delay_entries: if video_delay_entries or audio_delay_entries:
lines.append(f"- recommended video delays: {','.join(video_delay_entries)}") lines.append(f"- recommended video delays: {','.join(video_delay_entries)}")
lines.append(f"- recommended audio delays: {','.join(audio_delay_entries)}") lines.append(f"- recommended audio delays: {','.join(audio_delay_entries)}")
static_lines = [
f"Server-to-RC static calibration for {root}",
f"- verdict: {'ready' if summary['static_ready'] else 'needs-more-data'}",
(
"- criteria: "
f"min_runs={static_min_runs} "
f"max_spread_us={static_max_spread_us} "
f"max_p95_skew_ms={static_max_p95_skew_ms:.1f} "
f"max_median_skew_ms={static_max_median_skew_ms:.1f} "
f"require_freshness={static_require_freshness} "
f"require_smoothness={static_require_smoothness}"
),
]
for mode, entry in sorted(static_modes.items()):
marker = "READY" if entry.get("ready") else "HOLD"
reasons = "; ".join(entry.get("reasons") or [])
if not reasons and entry.get("ready"):
reasons = "all stability criteria met"
static_lines.append(
f"- {marker} {mode}: "
f"runs={entry.get('eligible_runs')}/{entry.get('total_runs')} "
f"video={entry.get('recommended_video_delay_us')}us "
f"audio={entry.get('recommended_audio_delay_us')}us "
f"spread={entry.get('tested_video_delay_spread_us')}us "
f"p95_max={entry.get('sync_p95_abs_skew_ms_max') or 0.0:.1f}ms "
f"median_abs_max={entry.get('sync_abs_median_skew_ms_max') or 0.0:.1f}ms "
f"freshness_budget_max={entry.get('freshness_budget_ms_max') or 0.0:.1f}ms"
)
static_lines.append(f" reason: {reasons}")
if static_video_entries or static_audio_entries:
static_lines.append(f"- static video delays: {','.join(static_video_entries)}")
static_lines.append(f"- static audio delays: {','.join(static_audio_entries)}")
lines.extend(["", *static_lines])
summary_txt.write_text("\n".join(lines) + "\n") summary_txt.write_text("\n".join(lines) + "\n")
static_txt.write_text("\n".join(static_lines) + "\n")
print("\n".join(lines)) print("\n".join(lines))
PY PY
} }
@ -1838,10 +2227,12 @@ fi
echo "==> server-to-RC mode matrix" echo "==> server-to-RC mode matrix"
echo " ↪ modes=${LESAVKA_SERVER_RC_MODES}" echo " ↪ modes=${LESAVKA_SERVER_RC_MODES}"
echo " ↪ mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}" echo " ↪ mode_source=${LESAVKA_SERVER_RC_MODE_SOURCE}"
echo " ↪ repeat_count=${LESAVKA_SERVER_RC_REPEAT_COUNT} verbose_probes=${LESAVKA_SERVER_RC_VERBOSE_PROBES}"
echo " ↪ video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}" echo " ↪ video_delays=${LESAVKA_SERVER_RC_MODE_DELAYS_US}"
echo " ↪ audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}" echo " ↪ audio_delays=${LESAVKA_SERVER_RC_MODE_AUDIO_DELAYS_US}"
echo " ↪ capture_stack=${REMOTE_CAPTURE_STACK} audio_source=${REMOTE_AUDIO_SOURCE} pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL} video_mode=${REMOTE_PULSE_VIDEO_MODE}" echo " ↪ capture_stack=${REMOTE_CAPTURE_STACK} audio_source=${REMOTE_AUDIO_SOURCE} pulse_tool=${REMOTE_PULSE_CAPTURE_TOOL} video_mode=${REMOTE_PULSE_VIDEO_MODE}"
echo " ↪ tune_delays=${LESAVKA_SERVER_RC_TUNE_DELAYS} confirm=${LESAVKA_SERVER_RC_TUNE_CONFIRM} min_pairs=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS} max_abs_skew_ms=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS} max_step_us=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US} min_change_us=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}" echo " ↪ tune_delays=${LESAVKA_SERVER_RC_TUNE_DELAYS} confirm=${LESAVKA_SERVER_RC_TUNE_CONFIRM} min_pairs=${LESAVKA_SERVER_RC_TUNE_MIN_PAIRS} max_abs_skew_ms=${LESAVKA_SERVER_RC_TUNE_MAX_ABS_SKEW_MS} max_step_us=${LESAVKA_SERVER_RC_TUNE_MAX_STEP_US} min_change_us=${LESAVKA_SERVER_RC_TUNE_MIN_CHANGE_US}"
echo " ↪ static_min_runs=${LESAVKA_SERVER_RC_STATIC_MIN_RUNS} static_max_spread_us=${LESAVKA_SERVER_RC_STATIC_MAX_SPREAD_US} static_max_p95_skew_ms=${LESAVKA_SERVER_RC_STATIC_MAX_P95_SKEW_MS} static_max_median_skew_ms=${LESAVKA_SERVER_RC_STATIC_MAX_MEDIAN_SKEW_MS}"
echo " ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS} min_pairs=${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}" echo " ↪ freshness_limit_ms=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS} min_pairs=${LESAVKA_SERVER_RC_FRESHNESS_MIN_PAIRS}"
echo " ↪ coded_pairs_min=${LESAVKA_SERVER_RC_MIN_CODED_PAIRS} require_all_coded=${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS} smoothness_gate=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS}" echo " ↪ coded_pairs_min=${LESAVKA_SERVER_RC_MIN_CODED_PAIRS} require_all_coded=${LESAVKA_SERVER_RC_REQUIRE_ALL_CODED_PAIRS} smoothness_gate=${LESAVKA_SERVER_RC_REQUIRE_SMOOTHNESS_PASS}"
echo " ↪ signal_ready=${LESAVKA_SERVER_RC_SIGNAL_READY} mode=${LESAVKA_SERVER_RC_SIGNAL_READY_MODE} attempts=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS} min_pairs=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} duration=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}s warmup=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}s retry_delay=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s" echo " ↪ signal_ready=${LESAVKA_SERVER_RC_SIGNAL_READY} mode=${LESAVKA_SERVER_RC_SIGNAL_READY_MODE} attempts=${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS} min_pairs=${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} duration=${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}s warmup=${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}s retry_delay=${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s"
@ -1850,144 +2241,164 @@ echo " ↪ reconfigure=${LESAVKA_SERVER_RC_RECONFIGURE} strategy=${LESAVKA_SER
echo " ↪ tethys_ready=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY} settle=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}s timeout=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}s preroll_discard=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}s" echo " ↪ tethys_ready=${LESAVKA_SERVER_RC_WAIT_TETHYS_READY} settle=${LESAVKA_SERVER_RC_TETHYS_SETTLE_SECONDS}s timeout=${LESAVKA_SERVER_RC_TETHYS_READY_TIMEOUT_SECONDS}s preroll_discard=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}s"
echo " ↪ start_delay=${LESAVKA_SERVER_RC_START_DELAY_SECONDS}s" echo " ↪ start_delay=${LESAVKA_SERVER_RC_START_DELAY_SECONDS}s"
echo " ↪ artifact_dir=${MATRIX_REPORT_DIR}" echo " ↪ artifact_dir=${MATRIX_REPORT_DIR}"
echo " ↪ matrix_run_log=${MATRIX_RUN_LOG}"
prime_remote_sudo prime_remote_sudo
sleep_start_delay sleep_start_delay
prebuild_probe_tools prebuild_probe_tools
if ! [[ "${LESAVKA_SERVER_RC_REPEAT_COUNT}" =~ ^[0-9]+$ ]] || (( LESAVKA_SERVER_RC_REPEAT_COUNT < 1 )); then
printf 'LESAVKA_SERVER_RC_REPEAT_COUNT must be a positive integer; got %s\n' "${LESAVKA_SERVER_RC_REPEAT_COUNT}" >&2
exit 64
fi
declare -A mode_run_counts=()
matrix_sequence=0
IFS=',' read -r -a modes <<<"${LESAVKA_SERVER_RC_MODES}" IFS=',' read -r -a modes <<<"${LESAVKA_SERVER_RC_MODES}"
for mode in "${modes[@]}"; do for mode in "${modes[@]}"; do
mode="${mode//[[:space:]]/}" mode="${mode//[[:space:]]/}"
[[ -n "${mode}" ]] || continue [[ -n "${mode}" ]] || continue
read -r width height fps < <(parse_mode "${mode}") read -r width height fps < <(parse_mode "${mode}")
video_delay_us="$(lookup_video_delay_us "${mode}")"
audio_delay_us="$(lookup_audio_delay_us "${mode}")"
id="$(mode_id "${mode}")"
mode_dir="${MATRIX_REPORT_DIR}/${id}"
mode_log="${mode_dir}/mode-run.log"
mode_result="${mode_dir}/mode-result.json"
seed_result="${mode_dir}/mode-result-seed.json"
readiness_dir="${mode_dir}/signal-readiness"
readiness_log="${readiness_dir}/signal-readiness-run.log"
readiness_attempts_json="${readiness_dir}/signal-readiness-attempts.json"
tuned_log="${mode_dir}/mode-tuned-run.log"
tuned_result="${mode_dir}/mode-result-tuned.json"
tune_env="${mode_dir}/mode-tune-candidate.env"
mkdir -p "${mode_dir}"
echo "==> mode ${mode}: video_delay_us=${video_delay_us} audio_delay_us=${audio_delay_us}" for repeat_index in $(seq 1 "${LESAVKA_SERVER_RC_REPEAT_COUNT}"); do
reconfigure_server_mode "${mode}" "${width}" "${height}" "${fps}" mode_run_counts["${mode}"]=$(( ${mode_run_counts["${mode}"]:-0} + 1 ))
wait_tethys_media_ready "${mode}" "${width}" "${height}" "${fps}" mode_run_index=${mode_run_counts["${mode}"]}
matrix_sequence=$((matrix_sequence + 1))
video_delay_us="$(lookup_video_delay_us "${mode}")"
audio_delay_us="$(lookup_audio_delay_us "${mode}")"
id_base="$(mode_id "${mode}")"
run_label="$(printf '%02d' "${mode_run_index}")"
id="${id_base}__run${run_label}"
mode_dir="${MATRIX_REPORT_DIR}/${id}"
mode_log="${mode_dir}/mode-run.log"
mode_result="${mode_dir}/mode-result.json"
seed_result="${mode_dir}/mode-result-seed.json"
readiness_dir="${mode_dir}/signal-readiness"
readiness_log="${readiness_dir}/signal-readiness-run.log"
readiness_attempts_json="${readiness_dir}/signal-readiness-attempts.json"
tuned_log="${mode_dir}/mode-tuned-run.log"
tuned_result="${mode_dir}/mode-result-tuned.json"
tune_env="${mode_dir}/mode-tune-candidate.env"
mkdir -p "${mode_dir}"
if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" && "${LESAVKA_SERVER_RC_SIGNAL_READY_MODE}" != "separate" ]]; then echo "==> mode ${mode} run ${mode_run_index} repeat ${repeat_index}/${LESAVKA_SERVER_RC_REPEAT_COUNT}: video_delay_us=${video_delay_us} audio_delay_us=${audio_delay_us}"
echo "==> mode ${mode}: using same-capture signal conditioning before measured probe" reconfigure_server_mode "${mode}" "${width}" "${height}" "${fps}"
fi wait_tethys_media_ready "${mode}" "${width}" "${height}" "${fps}"
if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" && "${LESAVKA_SERVER_RC_SIGNAL_READY_MODE}" == "separate" ]]; then if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" && "${LESAVKA_SERVER_RC_SIGNAL_READY_MODE}" != "separate" ]]; then
mkdir -p "${readiness_dir}" echo "==> mode ${mode} run ${mode_run_index}: using same-capture signal conditioning before measured probe"
echo "==> mode ${mode}: proving Tethys signal readiness before measured probe" fi
readiness_pass=1
readiness_status=0 if [[ "${LESAVKA_SERVER_RC_SIGNAL_READY}" != "0" && "${LESAVKA_SERVER_RC_SIGNAL_READY_MODE}" == "separate" ]]; then
readiness_artifact_dir="" mkdir -p "${readiness_dir}"
readiness_reason="no readiness attempts ran" echo "==> mode ${mode} run ${mode_run_index}: proving Tethys signal readiness before measured probe"
readiness_attempt_jsons=() readiness_pass=1
for readiness_attempt in $(seq 1 "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}"); do readiness_status=0
attempt_dir="${readiness_dir}/attempt-${readiness_attempt}" readiness_artifact_dir=""
readiness_log="${attempt_dir}/signal-readiness-run.log" readiness_reason="no readiness attempts ran"
readiness_attempt_json="${attempt_dir}/signal-readiness-attempt.json" readiness_attempt_jsons=()
mkdir -p "${attempt_dir}" for readiness_attempt in $(seq 1 "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}"); do
echo " ↪ readiness attempt ${readiness_attempt}/${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}: requiring ${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} paired coded events" attempt_dir="${readiness_dir}/attempt-${readiness_attempt}"
run_mode_probe \ readiness_log="${attempt_dir}/signal-readiness-run.log"
"${width}" \ readiness_attempt_json="${attempt_dir}/signal-readiness-attempt.json"
"${height}" \ mkdir -p "${attempt_dir}"
"${fps}" \ echo " ↪ readiness attempt ${readiness_attempt}/${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}: requiring ${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS} paired coded events"
"${audio_delay_us}" \ run_mode_probe \
"${video_delay_us}" \ "${width}" \
"${attempt_dir}" \ "${height}" \
"${readiness_log}" \ "${fps}" \
"${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}" \ "${audio_delay_us}" \
"${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}" \ "${video_delay_us}" \
"${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \ "${attempt_dir}" \
0 "${readiness_log}" \
readiness_status=${RUN_MODE_PROBE_STATUS} "${LESAVKA_SERVER_RC_SIGNAL_READY_DURATION_SECONDS}" \
readiness_artifact_dir="$(artifact_dir_from_log "${readiness_log}" "${attempt_dir}")" "${LESAVKA_SERVER_RC_SIGNAL_READY_WARMUP_SECONDS}" \
set +e "${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \
readiness_reason="$( 0
write_signal_readiness_attempt_result \ readiness_status=${RUN_MODE_PROBE_STATUS}
"${readiness_attempt}" \ readiness_artifact_dir="$(artifact_dir_from_log "${readiness_log}" "${attempt_dir}")"
"${readiness_artifact_dir}" \ set +e
readiness_reason="$(
write_signal_readiness_attempt_result \
"${readiness_attempt}" \
"${readiness_artifact_dir}" \
"${readiness_status}" \
"${readiness_log}" \
"${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \
"${readiness_attempt_json}" 2>&1
)"
readiness_pass=$?
set -e
readiness_attempt_jsons+=("${readiness_attempt_json}")
if [[ "${readiness_pass}" -eq 0 ]]; then
echo " ↪ readiness attempt ${readiness_attempt} passed: ${readiness_reason}"
break
fi
[[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed"
echo " ↪ readiness attempt ${readiness_attempt} failed: ${readiness_reason}"
if [[ "${readiness_attempt}" -lt "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}" ]]; then
echo " ↪ waiting ${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s before retrying signal readiness"
sleep "${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}"
fi
done
write_signal_readiness_attempts_summary "${readiness_attempts_json}" "${readiness_attempt_jsons[@]}"
if [[ "${readiness_pass}" -ne 0 ]]; then
[[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed"
echo " ↪ signal readiness failed: ${readiness_reason}"
write_signal_readiness_failure \
"${mode}" \
"${width}" \
"${height}" \
"${fps}" \
"${video_delay_us}" \
"${audio_delay_us}" \
"${readiness_status}" \ "${readiness_status}" \
"${readiness_log}" \ "${readiness_log}" \
"${LESAVKA_SERVER_RC_SIGNAL_READY_MIN_PAIRS}" \ "${readiness_artifact_dir}" \
"${readiness_attempt_json}" 2>&1 "${readiness_attempts_json}" \
)" "${readiness_reason}" \
readiness_pass=$? "${mode_result}"
set -e annotate_mode_result "${mode_result}" "${id_base}" "${mode_run_index}" "${repeat_index}" "${LESAVKA_SERVER_RC_REPEAT_COUNT}" "${matrix_sequence}" "readiness-failure"
readiness_attempt_jsons+=("${readiness_attempt_json}") cp "${mode_result}" "${seed_result}"
if [[ "${readiness_pass}" -eq 0 ]]; then if [[ "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then
echo " ↪ readiness attempt ${readiness_attempt} passed: ${readiness_reason}" break 2
break fi
continue
fi fi
[[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed" echo " ↪ signal readiness passed: artifact_dir=${readiness_artifact_dir}"
echo " ↪ readiness attempt ${readiness_attempt} failed: ${readiness_reason}"
if [[ "${readiness_attempt}" -lt "${LESAVKA_SERVER_RC_SIGNAL_READY_ATTEMPTS}" ]]; then
echo " ↪ waiting ${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}s before retrying signal readiness"
sleep "${LESAVKA_SERVER_RC_SIGNAL_READY_RETRY_DELAY_SECONDS}"
fi
done
write_signal_readiness_attempts_summary "${readiness_attempts_json}" "${readiness_attempt_jsons[@]}"
if [[ "${readiness_pass}" -ne 0 ]]; then
[[ -n "${readiness_reason}" ]] || readiness_reason="signal readiness failed"
echo " ↪ signal readiness failed: ${readiness_reason}"
write_signal_readiness_failure \
"${mode}" \
"${width}" \
"${height}" \
"${fps}" \
"${video_delay_us}" \
"${audio_delay_us}" \
"${readiness_status}" \
"${readiness_log}" \
"${readiness_artifact_dir}" \
"${readiness_attempts_json}" \
"${readiness_reason}" \
"${mode_result}"
cp "${mode_result}" "${seed_result}"
if [[ "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then
break
fi
continue
fi fi
echo " ↪ signal readiness passed: artifact_dir=${readiness_artifact_dir}"
fi
run_mode_probe "${width}" "${height}" "${fps}" "${audio_delay_us}" "${video_delay_us}" "${mode_dir}" "${mode_log}" echo "==> mode ${mode} run ${mode_run_index}: running seed probe"
run_status=${RUN_MODE_PROBE_STATUS} run_mode_probe "${width}" "${height}" "${fps}" "${audio_delay_us}" "${video_delay_us}" "${mode_dir}" "${mode_log}"
artifact_dir="$(artifact_dir_from_log "${mode_log}" "${mode_dir}")" run_status=${RUN_MODE_PROBE_STATUS}
write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${video_delay_us}" "${audio_delay_us}" "${run_status}" "${mode_log}" "${artifact_dir}" "${mode_result}" artifact_dir="$(artifact_dir_from_log "${mode_log}" "${mode_dir}")"
cp "${mode_result}" "${seed_result}" write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${video_delay_us}" "${audio_delay_us}" "${run_status}" "${mode_log}" "${artifact_dir}" "${mode_result}"
annotate_mode_result "${mode_result}" "${id_base}" "${mode_run_index}" "${repeat_index}" "${LESAVKA_SERVER_RC_REPEAT_COUNT}" "${matrix_sequence}" "seed"
cp "${mode_result}" "${seed_result}"
if [[ "${LESAVKA_SERVER_RC_TUNE_DELAYS}" != "0" && "${LESAVKA_SERVER_RC_TUNE_CONFIRM}" != "0" ]]; then if [[ "${LESAVKA_SERVER_RC_TUNE_DELAYS}" != "0" && "${LESAVKA_SERVER_RC_TUNE_CONFIRM}" != "0" ]]; then
write_tune_candidate_env "${seed_result}" "${tune_env}" write_tune_candidate_env "${seed_result}" "${tune_env}"
# shellcheck disable=SC1090 # shellcheck disable=SC1090
source "${tune_env}" source "${tune_env}"
if [[ "${tune_ready:-false}" == "true" ]]; then if [[ "${tune_ready:-false}" == "true" ]]; then
echo "==> mode ${mode}: confirming tuned delays video_delay_us=${tune_video_delay_us} audio_delay_us=${tune_audio_delay_us}" echo "==> mode ${mode} run ${mode_run_index}: confirming tuned delays video_delay_us=${tune_video_delay_us} audio_delay_us=${tune_audio_delay_us}"
echo " ↪ tune delta: video=${tune_video_delta_us}us audio=${tune_audio_delta_us}us pairs=${tune_paired_event_count} drift=${tune_drift_ms}ms" echo " ↪ tune delta: video=${tune_video_delta_us}us audio=${tune_audio_delta_us}us pairs=${tune_paired_event_count} drift=${tune_drift_ms}ms"
run_mode_probe "${width}" "${height}" "${fps}" "${tune_audio_delay_us}" "${tune_video_delay_us}" "${mode_dir}" "${tuned_log}" run_mode_probe "${width}" "${height}" "${fps}" "${tune_audio_delay_us}" "${tune_video_delay_us}" "${mode_dir}" "${tuned_log}"
tuned_status=${RUN_MODE_PROBE_STATUS} tuned_status=${RUN_MODE_PROBE_STATUS}
tuned_artifact_dir="$(artifact_dir_from_log "${tuned_log}" "${mode_dir}")" tuned_artifact_dir="$(artifact_dir_from_log "${tuned_log}" "${mode_dir}")"
write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${tune_video_delay_us}" "${tune_audio_delay_us}" "${tuned_status}" "${tuned_log}" "${tuned_artifact_dir}" "${tuned_result}" write_mode_result "${mode}" "${width}" "${height}" "${fps}" "${tune_video_delay_us}" "${tune_audio_delay_us}" "${tuned_status}" "${tuned_log}" "${tuned_artifact_dir}" "${tuned_result}"
cp "${tuned_result}" "${mode_result}" annotate_mode_result "${tuned_result}" "${id_base}" "${mode_run_index}" "${repeat_index}" "${LESAVKA_SERVER_RC_REPEAT_COUNT}" "${matrix_sequence}" "tuned"
run_status=${tuned_status} cp "${tuned_result}" "${mode_result}"
else run_status=${tuned_status}
echo " ↪ tune skipped: ${tune_reason:-not ready}" else
echo " ↪ tune skipped: ${tune_reason:-not ready}"
fi
fi fi
fi
if [[ "${run_status}" -ne 0 && "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then if [[ "${run_status}" -ne 0 && "${LESAVKA_SERVER_RC_CONTINUE_ON_FAIL}" == "0" ]]; then
break break 2
fi fi
done
done done
summarize_matrix summarize_matrix
@ -1999,6 +2410,11 @@ echo "mode_matrix_summary_csv: ${MATRIX_SUMMARY_CSV}"
echo "mode_matrix_summary_txt: ${MATRIX_SUMMARY_TXT}" echo "mode_matrix_summary_txt: ${MATRIX_SUMMARY_TXT}"
echo "mode_delay_recommendations_json: ${MATRIX_DELAY_JSON}" echo "mode_delay_recommendations_json: ${MATRIX_DELAY_JSON}"
echo "mode_delay_recommendations_env: ${MATRIX_DELAY_ENV}" echo "mode_delay_recommendations_env: ${MATRIX_DELAY_ENV}"
echo "mode_static_calibration_json: ${MATRIX_STATIC_JSON}"
echo "mode_static_calibration_csv: ${MATRIX_STATIC_CSV}"
echo "mode_static_calibration_txt: ${MATRIX_STATIC_TXT}"
echo "mode_static_calibration_env: ${MATRIX_STATIC_ENV}"
echo "mode_matrix_run_log: ${MATRIX_RUN_LOG}"
if python3 - <<'PY' "${MATRIX_SUMMARY_JSON}" if python3 - <<'PY' "${MATRIX_SUMMARY_JSON}"
import json import json

2
server/Cargo.toml
View File

@ -10,7 +10,7 @@ bench = false
[package] [package]
name = "lesavka_server" name = "lesavka_server"
version = "0.19.27" version = "0.19.28"
edition = "2024" edition = "2024"
autobins = false autobins = false

45
testing/tests/client_manual_sync_script_contract.rs
View File

@ -471,9 +471,13 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() {
"schema\": \"lesavka.server-rc-mode-delay-recommendations.v1\"", "schema\": \"lesavka.server-rc-mode-delay-recommendations.v1\"",
"output_delay_calibration", "output_delay_calibration",
"write_tune_candidate_env", "write_tune_candidate_env",
"annotate_mode_result",
"LESAVKA_SERVER_RC_REPEAT_COUNT=${LESAVKA_SERVER_RC_REPEAT_COUNT:-1}",
"mode-static-calibration.json",
"mode-matrix-run.log",
"mode-result-seed.json", "mode-result-seed.json",
"mode-result-tuned.json", "mode-result-tuned.json",
"==> mode ${mode}: confirming tuned delays", "==> mode ${mode} run ${mode_run_index}: confirming tuned delays",
"calibration_ready", "calibration_ready",
"calibration_video_target_offset_us", "calibration_video_target_offset_us",
"calibration_audio_target_offset_us", "calibration_audio_target_offset_us",
@ -484,25 +488,26 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() {
"signature_coverage", "signature_coverage",
"paired coded signatures", "paired coded signatures",
"signature_missing_codes", "signature_missing_codes",
"REMOTE_PULSE_CAPTURE_TOOL=\"${REMOTE_PULSE_CAPTURE_TOOL}\"", "probe_env=(",
"REMOTE_PULSE_VIDEO_MODE=\"${REMOTE_PULSE_VIDEO_MODE}\"", "\"REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL}\"",
"REMOTE_CAPTURE_STACK=\"${REMOTE_CAPTURE_STACK}\"", "\"REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE}\"",
"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=\"${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}\"", "\"REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK}\"",
"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=\"${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}\"", "\"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}\"",
"REMOTE_CAPTURE_READY_SETTLE_SECONDS=\"${REMOTE_CAPTURE_READY_SETTLE_SECONDS}\"", "\"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${LESAVKA_SERVER_RC_PREROLL_DISCARD_SECONDS}\"",
"PROBE_PREBUILD=0", "\"REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS}\"",
"VIDEO_SIZE=\"${width}x${height}\"", "\"PROBE_PREBUILD=0\"",
"VIDEO_FPS=\"${fps}\"", "\"VIDEO_SIZE=${width}x${height}\"",
"REMOTE_EXPECT_UVC_WIDTH=\"${width}\"", "\"VIDEO_FPS=${fps}\"",
"REMOTE_EXPECT_UVC_HEIGHT=\"${height}\"", "\"REMOTE_EXPECT_UVC_WIDTH=${width}\"",
"REMOTE_EXPECT_UVC_FPS=\"${fps}\"", "\"REMOTE_EXPECT_UVC_HEIGHT=${height}\"",
"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=\"${audio_delay_us}\"", "\"REMOTE_EXPECT_UVC_FPS=${fps}\"",
"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=\"${video_delay_us}\"", "\"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${audio_delay_us}\"",
"LESAVKA_OUTPUT_DELAY_APPLY=0", "\"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${video_delay_us}\"",
"LESAVKA_OUTPUT_DELAY_SAVE=0", "\"LESAVKA_OUTPUT_DELAY_APPLY=0\"",
"LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=\"${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}\"", "\"LESAVKA_OUTPUT_DELAY_SAVE=0\"",
"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=\"${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}\"", "\"LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_AGE_MS}\"",
"LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=\"${min_pairs}\"", "\"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_SERVER_RC_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}\"",
"\"LESAVKA_OUTPUT_FRESHNESS_MIN_PAIRS=${min_pairs}\"",
"sync did not pass", "sync did not pass",
"freshness did not pass", "freshness did not pass",
"video hiccups", "video hiccups",