fix: require fixed-delay sync confirmation

This commit is contained in:
Brad Stein 2026-05-03 19:47:08 -03:00
parent 1ce993b0e8
commit 67ede4390e
8 changed files with 148 additions and 11 deletions

View File

@ -105,6 +105,15 @@ path.
legacy calibration does not keep a hidden multi-second video delay alive.
- [x] Make the direct output probe report separate sync and clock-corrected
freshness verdicts from the same paired server-generated signatures.
- [x] Make applied direct output-delay calibrations run a fixed-delay
confirmation probe that must pass sync before the run can be trusted.
- [x] Refuse freshness passes when Theia/Tethys clock alignment is too uncertain
or would imply impossible negative media age.
- [ ] Keep UI/profile controls authoritative for UVC output profiles beyond
`640x480@20`; validate `1280x720@30` and `1920x1080@20/30` after sync is
locked.
- [ ] Keep the UI +/-5ms calibration nudges available as small post-baseline
operator trims for future non-probeable remote hosts.
- [x] Continue reporting client timing and sink handoff diagnostics from bundled packets.
- [ ] Add bundled-mode counters for first bundle, first audio push, first video feed,
dropped stale bundles, and bundle queue age.
@ -118,6 +127,8 @@ path.
- [ ] Focused server upstream-media tests including bundled stream acceptance.
- [x] Direct UVC/UAC probe can derive, gate, apply, and optionally save measured
output-delay calibration without using the fragile webcam-at-screen path.
- [x] Direct UVC/UAC probe confirms a newly applied static delay with a second
pass/fail sync run before moving on to freshness or smoothness claims.
- [x] Saved output-delay calibration is a static server-side baseline for the
UVC/UAC gadget path, not a dependency on probing every future attached host.
- [ ] Install on both ends and verify diagnostics show bundled webcam media.

6
Cargo.lock generated
View File

@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]]
name = "lesavka_client"
version = "0.19.6"
version = "0.19.7"
dependencies = [
"anyhow",
"async-stream",
@ -1686,7 +1686,7 @@ dependencies = [
[[package]]
name = "lesavka_common"
version = "0.19.6"
version = "0.19.7"
dependencies = [
"anyhow",
"base64",
@ -1698,7 +1698,7 @@ dependencies = [
[[package]]
name = "lesavka_server"
version = "0.19.6"
version = "0.19.7"
dependencies = [
"anyhow",
"base64",

View File

@ -4,7 +4,7 @@ path = "src/main.rs"
[package]
name = "lesavka_client"
version = "0.19.6"
version = "0.19.7"
edition = "2024"
[dependencies]

View File

@ -1,6 +1,6 @@
[package]
name = "lesavka_common"
version = "0.19.6"
version = "0.19.7"
edition = "2024"
build = "build.rs"

View File

@ -56,6 +56,7 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta
| `LESAVKA_CAPTURE_POWER_GRACE_SECS` | runtime/install/session override |
| `LESAVKA_CAPTURE_POWER_UNIT` | runtime/install/session override |
| `LESAVKA_CAPTURE_REMOTE` | runtime/install/session override |
| `LESAVKA_CLOCK_ALIGNMENT_SAMPLES` | manual direct UVC/UAC probe freshness trust gate; number of SSH midpoint clock samples per host, using the lowest-uncertainty sample, defaults to `5` |
| `LESAVKA_CLIENT_APP_SRC` | test/build contract variable; not runtime operator config |
| `LESAVKA_CLIENT_BUNDLE` | server installer output path for the generated client TLS enrollment bundle |
| `LESAVKA_CLIENT_CAPTURE_DIR` | client installer capture folder override; defaults to `~/Pictures/lesavka` |
@ -178,14 +179,17 @@ from `LESAVKA_CLIENT_PKI_SSH_SOURCE` over SSH. Runtime clients require the insta
| `LESAVKA_OUTPUT_DELAY_APPLY` | manual direct UVC/UAC probe override; apply the measured server output-delay correction through the calibration API when the probe gates pass |
| `LESAVKA_OUTPUT_DELAY_APPLY_MODE` | manual direct UVC/UAC probe override; `absolute` sets the active output-path baseline to the measured device delay, while `relative` preserves legacy nudge behavior |
| `LESAVKA_OUTPUT_DELAY_CALIBRATION` | manual direct UVC/UAC probe override; emit `output-delay-calibration.json` from a lab-attached USB host capture of server-generated signatures, defaults to enabled |
| `LESAVKA_OUTPUT_DELAY_CONFIRM` | manual direct UVC/UAC probe safety gate; after applying a ready output-delay measurement, rerun a fixed-delay confirmation probe that must pass sync, defaults to enabled |
| `LESAVKA_OUTPUT_DELAY_GAIN` | manual direct UVC/UAC probe override; scales measured output-delay correction before applying, defaults to `1.0` |
| `LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS` | manual direct UVC/UAC probe safety limit; refuses to apply/save implausibly large measured device skew, defaults to `5000` |
| `LESAVKA_OUTPUT_DELAY_MAX_DRIFT_MS` | manual direct UVC/UAC probe stability limit; refuses to apply/save unstable output-delay measurements, defaults to `80` |
| `LESAVKA_OUTPUT_DELAY_MAX_STEP_US` | manual direct UVC/UAC probe safety limit; clamps one measured correction step, defaults to `1500000` |
| `LESAVKA_OUTPUT_DELAY_MIN_PAIRS` | manual direct UVC/UAC probe evidence floor before applying measured output-delay calibration, defaults to `8` |
| `LESAVKA_OUTPUT_REQUIRE_SYNC_PASS` | manual direct UVC/UAC probe safety gate; fail the run unless the analyzer verdict passes sync, used by fixed-delay confirmation |
| `LESAVKA_OUTPUT_DELAY_SAVE` | manual direct UVC/UAC probe override; after applying a ready measured correction, persist it as the server default calibration |
| `LESAVKA_OUTPUT_DELAY_TARGET` | manual direct UVC/UAC probe override; choose whether measured skew is corrected by shifting `video` or `audio`, defaults to `video` |
| `LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS` | manual direct UVC/UAC probe freshness gate; maximum clock-corrected server-feed-to-Tethys-observed p95 age, defaults to `1000` |
| `LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS` | manual direct UVC/UAC probe freshness trust gate; do not pass freshness when host clock alignment uncertainty exceeds this, defaults to `100` |
| `LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS` | manual direct UVC/UAC probe freshness gate; maximum allowed freshness drift across paired probe events, defaults to `100` |
| `LESAVKA_PASTE_DELAY_MS` | input routing/clipboard override |
| `LESAVKA_PASTE_KEY` | input routing/clipboard override |

View File

@ -56,7 +56,9 @@ REMOTE_EXPECT_UVC_CODEC=${REMOTE_EXPECT_UVC_CODEC:-mjpeg}
LESAVKA_OUTPUT_DELAY_CALIBRATION=${LESAVKA_OUTPUT_DELAY_CALIBRATION:-1}
LESAVKA_OUTPUT_DELAY_APPLY=${LESAVKA_OUTPUT_DELAY_APPLY:-0}
LESAVKA_OUTPUT_DELAY_APPLY_MODE=${LESAVKA_OUTPUT_DELAY_APPLY_MODE:-absolute}
LESAVKA_OUTPUT_DELAY_CONFIRM=${LESAVKA_OUTPUT_DELAY_CONFIRM:-1}
LESAVKA_OUTPUT_DELAY_SAVE=${LESAVKA_OUTPUT_DELAY_SAVE:-0}
LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=${LESAVKA_OUTPUT_REQUIRE_SYNC_PASS:-0}
LESAVKA_OUTPUT_DELAY_TARGET=${LESAVKA_OUTPUT_DELAY_TARGET:-video}
LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${LESAVKA_OUTPUT_DELAY_MIN_PAIRS:-8}
LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS:-5000}
@ -65,6 +67,8 @@ LESAVKA_OUTPUT_DELAY_GAIN=${LESAVKA_OUTPUT_DELAY_GAIN:-1.0}
LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_OUTPUT_DELAY_MAX_STEP_US:-1500000}
LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS:-1000}
LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS:-100}
LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-100}
LESAVKA_CLOCK_ALIGNMENT_SAMPLES=${LESAVKA_CLOCK_ALIGNMENT_SAMPLES:-5}
CAPTURE_READY_MARKER="__LESAVKA_CAPTURE_READY__"
STAMP="$(date +%Y%m%d-%H%M%S)"
@ -126,15 +130,46 @@ print(f"{remote_ns - local_mid_ns} {(after_ns - before_ns) // 2} {after_ns - bef
PY
}
sample_best_host_clock_offset_ns() {
local host=$1
local samples=$2
local tmp
tmp="$(mktemp)"
local i=0
while (( i < samples )); do
sample_host_clock_offset_ns "${host}" >>"${tmp}" 2>/dev/null || true
((i += 1))
done
python3 - <<'PY' "${tmp}"
import pathlib
import sys
rows = []
for line in pathlib.Path(sys.argv[1]).read_text().splitlines():
try:
offset_ns, uncertainty_ns, rtt_ns = (int(value) for value in line.split())
except Exception:
continue
rows.append((uncertainty_ns, offset_ns, rtt_ns))
if not rows:
raise SystemExit(1)
uncertainty_ns, offset_ns, rtt_ns = min(rows)
print(f"{offset_ns} {uncertainty_ns} {rtt_ns} {len(rows)}")
PY
local rc=$?
rm -f "${tmp}"
return "${rc}"
}
write_clock_alignment() {
echo "==> sampling Theia/Tethys clock alignment for freshness"
local theia_sample tethys_sample
if ! theia_sample="$(sample_host_clock_offset_ns "${LESAVKA_SERVER_HOST}")"; then
if ! theia_sample="$(sample_best_host_clock_offset_ns "${LESAVKA_SERVER_HOST}" "${LESAVKA_CLOCK_ALIGNMENT_SAMPLES}")"; then
echo " ↪ clock alignment unavailable: failed to sample ${LESAVKA_SERVER_HOST}"
printf '{"schema":"lesavka.clock-alignment.v1","available":false,"reason":"failed to sample server host"}\n' >"${LOCAL_CLOCK_ALIGNMENT_JSON}"
return 0
fi
if ! tethys_sample="$(sample_host_clock_offset_ns "${TETHYS_HOST}")"; then
if ! tethys_sample="$(sample_best_host_clock_offset_ns "${TETHYS_HOST}" "${LESAVKA_CLOCK_ALIGNMENT_SAMPLES}")"; then
echo " ↪ clock alignment unavailable: failed to sample ${TETHYS_HOST}"
printf '{"schema":"lesavka.clock-alignment.v1","available":false,"reason":"failed to sample capture host"}\n' >"${LOCAL_CLOCK_ALIGNMENT_JSON}"
return 0
@ -151,8 +186,8 @@ import pathlib
import sys
server_host, capture_host, server_sample, capture_sample, output_path = sys.argv[1:]
server_offset_ns, server_uncertainty_ns, server_rtt_ns = (int(value) for value in server_sample.split())
capture_offset_ns, capture_uncertainty_ns, capture_rtt_ns = (int(value) for value in capture_sample.split())
server_offset_ns, server_uncertainty_ns, server_rtt_ns, server_samples = (int(value) for value in server_sample.split())
capture_offset_ns, capture_uncertainty_ns, capture_rtt_ns, capture_samples = (int(value) for value in capture_sample.split())
theia_to_tethys_offset_ns = capture_offset_ns - server_offset_ns
uncertainty_ns = server_uncertainty_ns + capture_uncertainty_ns
artifact = {
@ -168,6 +203,8 @@ artifact = {
"uncertainty_ms": uncertainty_ns / 1_000_000.0,
"server_sample_rtt_ns": server_rtt_ns,
"capture_sample_rtt_ns": capture_rtt_ns,
"server_samples": server_samples,
"capture_samples": capture_samples,
}
pathlib.Path(output_path).write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n")
print(
@ -544,7 +581,8 @@ write_output_delay_correlation() {
"${LOCAL_CAPTURE_LOG}" \
"${LOCAL_CLOCK_ALIGNMENT_JSON}" \
"${LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS}" \
"${LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS}"
"${LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS}" \
"${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS}"
import csv
import json
import math
@ -561,6 +599,7 @@ import sys
clock_alignment_path,
max_freshness_age_raw,
max_freshness_drift_raw,
max_clock_uncertainty_raw,
) = sys.argv[1:]
report = json.loads(pathlib.Path(report_path).read_text())
timeline = json.loads(pathlib.Path(timeline_path).read_text())
@ -821,6 +860,7 @@ correction_mode = (
max_freshness_age_ms = max(1.0, as_float(max_freshness_age_raw, 1000.0))
max_freshness_drift_ms = max(0.0, as_float(max_freshness_drift_raw, 100.0))
max_clock_uncertainty_ms = max(0.0, as_float(max_clock_uncertainty_raw, 100.0))
freshness_p95_values = [
value
for value in [
@ -842,6 +882,19 @@ freshness_worst_drift_ms = max(freshness_drift_values) if freshness_drift_values
if not freshness_p95_values:
freshness_status = "unknown"
freshness_reason = "clock-aligned server feed and Tethys capture timestamps were not available"
elif not clock_alignment_available or clock_uncertainty_ms > max_clock_uncertainty_ms:
freshness_status = "unknown"
freshness_reason = (
f"clock uncertainty {clock_uncertainty_ms:.1f} ms exceeds "
f"{max_clock_uncertainty_ms:.1f} ms freshness measurement limit"
)
elif freshness_worst_p95_ms < -clock_uncertainty_ms:
freshness_status = "invalid"
freshness_reason = (
f"freshness was negative beyond clock uncertainty: "
f"worst p95 {freshness_worst_p95_ms:.1f} ms, uncertainty "
f"{clock_uncertainty_ms:.1f} ms"
)
elif freshness_worst_p95_ms <= max_freshness_age_ms and (
freshness_worst_drift_ms is None or freshness_worst_drift_ms <= max_freshness_drift_ms
):
@ -880,6 +933,7 @@ artifact = {
"clock_uncertainty_ms": clock_uncertainty_ms,
"max_age_limit_ms": max_freshness_age_ms,
"max_drift_limit_ms": max_freshness_drift_ms,
"max_clock_uncertainty_ms": max_clock_uncertainty_ms,
"worst_p95_freshness_ms": freshness_worst_p95_ms,
"worst_freshness_drift_ms": freshness_worst_drift_ms,
"video_freshness_stats": video_freshness_stats,
@ -1047,6 +1101,59 @@ maybe_apply_output_delay_calibration() {
fi
}
maybe_run_output_delay_confirmation() {
[[ "${LESAVKA_OUTPUT_DELAY_CONFIRM}" != "0" ]] || return 0
[[ "${LESAVKA_OUTPUT_DELAY_APPLY}" != "0" ]] || return 0
[[ "${LESAVKA_OUTPUT_DELAY_CONFIRMING:-0}" != "1" ]] || return 0
[[ "${output_delay_ready:-false}" == "true" ]] || return 0
local confirm_audio_delay="${output_delay_audio_target_offset_us:-0}"
local confirm_video_delay="${output_delay_video_target_offset_us:-0}"
local confirm_output_dir="${LOCAL_REPORT_DIR}/confirmation"
mkdir -p "${confirm_output_dir}"
echo "==> confirming fixed UVC/UAC output-delay calibration"
echo " ↪ confirmation_audio_delay_us=${confirm_audio_delay}"
echo " ↪ confirmation_video_delay_us=${confirm_video_delay}"
echo " ↪ confirmation_requires_sync_pass=1"
LESAVKA_OUTPUT_DELAY_CONFIRMING=1 \
LESAVKA_OUTPUT_DELAY_CONFIRM=0 \
LESAVKA_OUTPUT_DELAY_APPLY=0 \
LESAVKA_OUTPUT_DELAY_SAVE=0 \
LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=1 \
LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US="${confirm_audio_delay}" \
LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US="${confirm_video_delay}" \
PROBE_PREBUILD=0 \
LOCAL_OUTPUT_DIR="${confirm_output_dir}" \
"${SCRIPT_DIR}/run_upstream_av_sync.sh"
}
enforce_sync_verdict() {
[[ "${LESAVKA_OUTPUT_REQUIRE_SYNC_PASS}" != "0" ]] || return 0
[[ -f "${LOCAL_ANALYSIS_JSON}" ]] || {
echo "required sync pass unavailable: missing ${LOCAL_ANALYSIS_JSON}" >&2
exit 94
}
python3 - <<'PY' "${LOCAL_ANALYSIS_JSON}"
import json
import pathlib
import sys
report = json.loads(pathlib.Path(sys.argv[1]).read_text())
verdict = report.get("verdict") or {}
if verdict.get("passed") is True:
raise SystemExit(0)
print(
"required sync pass failed: "
f"{verdict.get('status', 'unknown')} - {verdict.get('reason', '')}",
file=sys.stderr,
)
raise SystemExit(94)
PY
}
if [[ "${PROBE_PREBUILD}" != "0" ]]; then
echo "==> prebuilding relay control/analyzer before opening the capture window"
(
@ -1795,6 +1902,8 @@ fi
write_output_delay_correlation
write_output_delay_calibration
maybe_apply_output_delay_calibration
maybe_run_output_delay_confirmation
enforce_sync_verdict
if [[ "${capture_v4l2_fault}" -eq 1 ]]; then
echo "warning: Tethys video capture reported VIDIOC_QBUF / Bad file descriptor; treat unstable skew or analyzer failures as host-capture suspect" >&2

View File

@ -10,7 +10,7 @@ bench = false
[package]
name = "lesavka_server"
version = "0.19.6"
version = "0.19.7"
edition = "2024"
autobins = false

View File

@ -43,7 +43,9 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
"LESAVKA_OUTPUT_DELAY_CALIBRATION=${LESAVKA_OUTPUT_DELAY_CALIBRATION:-1}",
"LESAVKA_OUTPUT_DELAY_APPLY=${LESAVKA_OUTPUT_DELAY_APPLY:-0}",
"LESAVKA_OUTPUT_DELAY_APPLY_MODE=${LESAVKA_OUTPUT_DELAY_APPLY_MODE:-absolute}",
"LESAVKA_OUTPUT_DELAY_CONFIRM=${LESAVKA_OUTPUT_DELAY_CONFIRM:-1}",
"LESAVKA_OUTPUT_DELAY_SAVE=${LESAVKA_OUTPUT_DELAY_SAVE:-0}",
"LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=${LESAVKA_OUTPUT_REQUIRE_SYNC_PASS:-0}",
"LESAVKA_OUTPUT_DELAY_TARGET=${LESAVKA_OUTPUT_DELAY_TARGET:-video}",
"LESAVKA_OUTPUT_DELAY_MIN_PAIRS=${LESAVKA_OUTPUT_DELAY_MIN_PAIRS:-8}",
"LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS=${LESAVKA_OUTPUT_DELAY_MAX_ABS_SKEW_MS:-5000}",
@ -51,6 +53,9 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
"LESAVKA_OUTPUT_DELAY_MAX_STEP_US=${LESAVKA_OUTPUT_DELAY_MAX_STEP_US:-1500000}",
"LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_AGE_MS:-1000}",
"LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_DRIFT_MS:-100}",
"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-100}",
"LESAVKA_CLOCK_ALIGNMENT_SAMPLES=${LESAVKA_CLOCK_ALIGNMENT_SAMPLES:-5}",
"sample_best_host_clock_offset_ns",
"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US:-0}",
"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US:-0}",
"write_clock_alignment",
@ -59,6 +64,8 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
"extract_server_timeline",
"write_output_delay_correlation",
"maybe_apply_output_delay_calibration",
"maybe_run_output_delay_confirmation",
"enforce_sync_verdict",
"schema\": \"lesavka.output-delay-calibration.v1\"",
"schema\": \"lesavka.output-delay-correlation.v1\"",
"schema\": \"lesavka.clock-alignment.v1\"",
@ -82,6 +89,12 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
"video_target_offset_us",
"output_delay_audio_target_offset_us",
"output_delay_video_target_offset_us",
"LESAVKA_OUTPUT_DELAY_CONFIRMING=1",
"LESAVKA_OUTPUT_REQUIRE_SYNC_PASS=1",
"LESAVKA_OUTPUT_DELAY_PROBE_AUDIO_DELAY_US=\"${confirm_audio_delay}\"",
"LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US=\"${confirm_video_delay}\"",
"==> confirming fixed UVC/UAC output-delay calibration",
"required sync pass failed",
"calibration_active_video_offset_us",
"absolute_target_video_offset_us",
"calibration_apply_video_delta_us",