test(server-rc): harden output freshness proof

This commit is contained in:
Brad Stein 2026-05-04 18:47:22 -03:00
parent 1e343057ac
commit b4587970b4
8 changed files with 460 additions and 77 deletions

6
Cargo.lock generated
View File

@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]] [[package]]
name = "lesavka_client" name = "lesavka_client"
version = "0.19.21" version = "0.19.22"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-stream", "async-stream",
@ -1686,7 +1686,7 @@ dependencies = [
[[package]] [[package]]
name = "lesavka_common" name = "lesavka_common"
version = "0.19.21" version = "0.19.22"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"base64", "base64",
@ -1698,7 +1698,7 @@ dependencies = [
[[package]] [[package]]
name = "lesavka_server" name = "lesavka_server"
version = "0.19.21" version = "0.19.22"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"base64", "base64",

View File

@ -4,7 +4,7 @@ path = "src/main.rs"
[package] [package]
name = "lesavka_client" name = "lesavka_client"
version = "0.19.21" version = "0.19.22"
edition = "2024" edition = "2024"
[dependencies] [dependencies]

View File

@ -1,6 +1,6 @@
[package] [package]
name = "lesavka_common" name = "lesavka_common"
version = "0.19.21" version = "0.19.22"
edition = "2024" edition = "2024"
build = "build.rs" build = "build.rs"

View File

@ -910,7 +910,9 @@ artifact_dir = pathlib.Path(artifact_dir_raw) if artifact_dir_raw else pathlib.P
report = load_json(artifact_dir / "report.json") report = load_json(artifact_dir / "report.json")
correlation = load_json(artifact_dir / "output-delay-correlation.json") correlation = load_json(artifact_dir / "output-delay-correlation.json")
calibration = load_json(artifact_dir / "output-delay-calibration.json") calibration = load_json(artifact_dir / "output-delay-calibration.json")
timing_map = correlation.get("timing_map") or {}
freshness = correlation.get("freshness") or {} freshness = correlation.get("freshness") or {}
capture_timebase = freshness.get("capture_timebase") or {}
smoothness = correlation.get("smoothness") or {} smoothness = correlation.get("smoothness") or {}
video = smoothness.get("video") or {} video = smoothness.get("video") or {}
audio = smoothness.get("audio") or {} audio = smoothness.get("audio") or {}
@ -972,6 +974,11 @@ if as_bool(require_sync_raw) and not sync_pass:
reasons.append(f"sync did not pass: {verdict.get('status', 'unknown')}") reasons.append(f"sync did not pass: {verdict.get('status', 'unknown')}")
if as_bool(require_freshness_raw) and not freshness_pass: if as_bool(require_freshness_raw) and not freshness_pass:
reasons.append(f"freshness did not pass: {freshness_status}") reasons.append(f"freshness did not pass: {freshness_status}")
if capture_timebase and capture_timebase.get("valid") is False:
reasons.append(
"capture timebase invalid: "
f"{capture_timebase.get('reason', capture_timebase.get('status', 'invalid'))}"
)
if signature_expected > 0: if signature_expected > 0:
if signature_paired < signature_expected: if signature_paired < signature_expected:
reasons.append(f"paired coded signatures {signature_paired} < expected {signature_expected}") reasons.append(f"paired coded signatures {signature_paired} < expected {signature_expected}")
@ -1010,6 +1017,7 @@ artifact = {
"report_json": str(artifact_dir / "report.json"), "report_json": str(artifact_dir / "report.json"),
"correlation_json": str(artifact_dir / "output-delay-correlation.json"), "correlation_json": str(artifact_dir / "output-delay-correlation.json"),
"calibration_json": str(artifact_dir / "output-delay-calibration.json"), "calibration_json": str(artifact_dir / "output-delay-calibration.json"),
"timing_map": timing_map,
"passed": not reasons, "passed": not reasons,
"failure_reasons": reasons, "failure_reasons": reasons,
"sync": { "sync": {
@ -1037,10 +1045,15 @@ artifact = {
"status": freshness_status, "status": freshness_status,
"reason": freshness.get("reason", ""), "reason": freshness.get("reason", ""),
"worst_event_age_p95_ms": freshness.get("worst_event_age_p95_ms"), "worst_event_age_p95_ms": freshness.get("worst_event_age_p95_ms"),
"worst_p95_pipeline_freshness_ms": freshness.get("worst_p95_pipeline_freshness_ms"),
"minimum_pipeline_freshness_ms": freshness.get("minimum_pipeline_freshness_ms"),
"clock_uncertainty_ms": freshness.get("clock_uncertainty_ms"), "clock_uncertainty_ms": freshness.get("clock_uncertainty_ms"),
"worst_event_age_with_uncertainty_ms": freshness.get("worst_event_age_with_uncertainty_ms"), "worst_event_age_with_uncertainty_ms": freshness.get("worst_event_age_with_uncertainty_ms"),
"worst_freshness_drift_ms": freshness.get("worst_freshness_drift_ms"), "worst_freshness_drift_ms": freshness.get("worst_freshness_drift_ms"),
"max_age_limit_ms": freshness.get("max_age_limit_ms"), "max_age_limit_ms": freshness.get("max_age_limit_ms"),
"capture_timebase_status": capture_timebase.get("status"),
"capture_timebase_valid": capture_timebase.get("valid"),
"capture_timebase_reason": capture_timebase.get("reason"),
}, },
"smoothness": { "smoothness": {
"video_frames": as_int(video.get("timestamps"), 0), "video_frames": as_int(video.get("timestamps"), 0),
@ -1221,6 +1234,7 @@ fieldnames = [
"freshness_status", "freshness_status",
"freshness_budget_ms", "freshness_budget_ms",
"freshness_drift_ms", "freshness_drift_ms",
"capture_timebase_status",
"video_hiccups", "video_hiccups",
"video_missing", "video_missing",
"video_undecodable", "video_undecodable",
@ -1252,6 +1266,7 @@ with summary_csv.open("w", newline="", encoding="utf-8") as handle:
"freshness_status": (result.get("freshness") or {}).get("status"), "freshness_status": (result.get("freshness") or {}).get("status"),
"freshness_budget_ms": (result.get("freshness") or {}).get("worst_event_age_with_uncertainty_ms"), "freshness_budget_ms": (result.get("freshness") or {}).get("worst_event_age_with_uncertainty_ms"),
"freshness_drift_ms": (result.get("freshness") or {}).get("worst_freshness_drift_ms"), "freshness_drift_ms": (result.get("freshness") or {}).get("worst_freshness_drift_ms"),
"capture_timebase_status": (result.get("freshness") or {}).get("capture_timebase_status"),
"video_hiccups": (result.get("smoothness") or {}).get("video_hiccups"), "video_hiccups": (result.get("smoothness") or {}).get("video_hiccups"),
"video_missing": (result.get("smoothness") or {}).get("video_estimated_missing_frames"), "video_missing": (result.get("smoothness") or {}).get("video_estimated_missing_frames"),
"video_undecodable": (result.get("smoothness") or {}).get("video_undecodable_frames"), "video_undecodable": (result.get("smoothness") or {}).get("video_undecodable_frames"),
@ -1301,6 +1316,13 @@ for result in results:
f"pair_conf_median={sync.get('paired_confidence_median', 0.0):.3f} " f"pair_conf_median={sync.get('paired_confidence_median', 0.0):.3f} "
f"raw_pair_disagreement={sync.get('activity_pair_disagreement_ms', 0.0):+.1f}ms" f"raw_pair_disagreement={sync.get('activity_pair_disagreement_ms', 0.0):+.1f}ms"
) )
if freshness.get("capture_timebase_status"):
lines.append(
" capture timing: "
f"{freshness.get('capture_timebase_status')} "
f"valid={freshness.get('capture_timebase_valid')} "
f"reason={freshness.get('capture_timebase_reason', '')}"
)
if "seed_video_delay_us" in result or "seed_audio_delay_us" in result: if "seed_video_delay_us" in result or "seed_audio_delay_us" in result:
lines.append( lines.append(
" tuning: " " tuning: "

View File

@ -40,6 +40,7 @@ VIDEO_FORMAT=${VIDEO_FORMAT:-mjpeg}
REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse} REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse}
REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst} REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}
REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-copy} REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-copy}
REMOTE_PULSE_AUDIO_ANCHOR_SILENCE=${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE:-1}
REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto} REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto}
REMOTE_AUDIO_QUIESCE_USER_AUDIO=${REMOTE_AUDIO_QUIESCE_USER_AUDIO:-auto} REMOTE_AUDIO_QUIESCE_USER_AUDIO=${REMOTE_AUDIO_QUIESCE_USER_AUDIO:-auto}
REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0} REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}
@ -729,6 +730,12 @@ def env_line(key, value):
report = json.loads(pathlib.Path(report_path).read_text()) report = json.loads(pathlib.Path(report_path).read_text())
verdict = report.get("verdict") or {} verdict = report.get("verdict") or {}
try:
correlation = json.loads(pathlib.Path(correlation_path).read_text())
except Exception:
correlation = {}
freshness = correlation.get("freshness") or {}
capture_timebase = freshness.get("capture_timebase") or {}
target = target.strip().lower() target = target.strip().lower()
apply_mode = apply_mode_raw.strip().lower() apply_mode = apply_mode_raw.strip().lower()
@ -777,6 +784,15 @@ if max_abs_observed_ms > max_abs_skew_ms:
) )
if abs(drift_ms) > max_drift_ms: if abs(drift_ms) > max_drift_ms:
refusal_reasons.append(f"abs(drift_ms) {abs(drift_ms):.1f} > {max_drift_ms:.1f}") refusal_reasons.append(f"abs(drift_ms) {abs(drift_ms):.1f} > {max_drift_ms:.1f}")
if capture_timebase and capture_timebase.get("valid") is not True:
refusal_reasons.append(
"capture timebase invalid for calibration: "
f"{capture_timebase.get('status', 'unknown')} - {capture_timebase.get('reason', '')}"
)
elif freshness.get("status") == "invalid":
refusal_reasons.append(
f"freshness invalid for calibration: {freshness.get('reason', '')}"
)
audio_target_offset_us = active_audio_offset_us + audio_delta_us audio_target_offset_us = active_audio_offset_us + audio_delta_us
video_target_offset_us = active_video_offset_us + video_delta_us video_target_offset_us = active_video_offset_us + video_delta_us
@ -801,6 +817,10 @@ artifact = {
"measurement_host_role": "lab-attached USB host", "measurement_host_role": "lab-attached USB host",
"probe_media_origin": "server-generated", "probe_media_origin": "server-generated",
"probe_media_path": "server generated signatures -> UVC/UAC sinks -> lab host capture", "probe_media_path": "server generated signatures -> UVC/UAC sinks -> lab host capture",
"injection_scope": "server-final-output-handoff",
"server_pipeline_reference": "generated media PTS before intentional sync delay",
"sink_handoff_path": "video CameraRelay::feed; audio Voice::push",
"client_uplink_included": False,
"report_json": report_path, "report_json": report_path,
"correlation_json": correlation_path if pathlib.Path(correlation_path).exists() else "", "correlation_json": correlation_path if pathlib.Path(correlation_path).exists() else "",
"audio_after_video_positive": True, "audio_after_video_positive": True,
@ -1444,6 +1464,27 @@ def analyze_smoothness(path, report, timeline):
video_timestamps = ffprobe_frame_timestamps(path) video_timestamps = ffprobe_frame_timestamps(path)
audio_packets = ffprobe_packet_times(path, "a:0") audio_packets = ffprobe_packet_times(path, "a:0")
audio_packet_timestamps = [pts for pts, _duration in audio_packets] audio_packet_timestamps = [pts for pts, _duration in audio_packets]
media_timeline = {
"video_first_s": min(video_timestamps) if video_timestamps else None,
"video_last_s": max(video_timestamps) if video_timestamps else None,
"audio_first_s": min(audio_packet_timestamps) if audio_packet_timestamps else None,
"audio_last_s": max(audio_packet_timestamps) if audio_packet_timestamps else None,
}
media_timeline["video_span_s"] = (
media_timeline["video_last_s"] - media_timeline["video_first_s"]
if media_timeline["video_first_s"] is not None and media_timeline["video_last_s"] is not None
else None
)
media_timeline["audio_span_s"] = (
media_timeline["audio_last_s"] - media_timeline["audio_first_s"]
if media_timeline["audio_first_s"] is not None and media_timeline["audio_last_s"] is not None
else None
)
media_timeline["audio_video_span_gap_s"] = (
media_timeline["video_span_s"] - media_timeline["audio_span_s"]
if media_timeline["video_span_s"] is not None and media_timeline["audio_span_s"] is not None
else None
)
video_ids = continuity_frame_ids(path) video_ids = continuity_frame_ids(path)
if window_start_s is not None and window_end_s is not None and len(video_ids) == len(video_timestamps): if window_start_s is not None and window_end_s is not None and len(video_ids) == len(video_timestamps):
video_ids = [ video_ids = [
@ -1464,6 +1505,7 @@ def analyze_smoothness(path, report, timeline):
"scope": "captured RC target media cadence and continuity over the server-generated probe window", "scope": "captured RC target media cadence and continuity over the server-generated probe window",
"window_start_s": window_start_s, "window_start_s": window_start_s,
"window_end_s": window_end_s, "window_end_s": window_end_s,
"media_timeline": media_timeline,
"video": { "video": {
**interval_smoothness(video_timestamps, expected_video_ms, window_start_s, window_end_s), **interval_smoothness(video_timestamps, expected_video_ms, window_start_s, window_end_s),
**sequence_smoothness(video_ids), **sequence_smoothness(video_ids),
@ -1504,6 +1546,22 @@ def corrected_server_capture_time_s(server, key):
return (server_unix_ns + theia_to_tethys_offset_ns - capture_start_unix_ns) / 1_000_000_000.0 return (server_unix_ns + theia_to_tethys_offset_ns - capture_start_unix_ns) / 1_000_000_000.0
def corrected_capture_time_s_from_unix_ns(server_unix_ns):
if (
server_unix_ns is None
or theia_to_tethys_offset_ns is None
or capture_start_unix_ns is None
):
return None
return (server_unix_ns + theia_to_tethys_offset_ns - capture_start_unix_ns) / 1_000_000_000.0
def shifted_unix_ns(unix_ns, delta_ms):
if unix_ns is None:
return None
return unix_ns + int(round(delta_ms * 1_000_000.0))
def nearest_server_event_for_observation(observed, used_event_ids): def nearest_server_event_for_observation(observed, used_event_ids):
observed_video_s = finite(observed.get("video_time_s")) observed_video_s = finite(observed.get("video_time_s"))
observed_audio_s = finite(observed.get("audio_time_s")) observed_audio_s = finite(observed.get("audio_time_s"))
@ -1573,6 +1631,14 @@ for observed in report.get("paired_events", []):
server_audio_push_s /= 1_000_000.0 server_audio_push_s /= 1_000_000.0
server_video_feed_unix_ns = as_int_or_none(server.get("video_feed_unix_ns")) server_video_feed_unix_ns = as_int_or_none(server.get("video_feed_unix_ns"))
server_audio_push_unix_ns = as_int_or_none(server.get("audio_push_unix_ns")) server_audio_push_unix_ns = as_int_or_none(server.get("audio_push_unix_ns"))
server_video_pipeline_unix_ns = shifted_unix_ns(
server_video_feed_unix_ns,
-video_delay_ms,
)
server_audio_pipeline_unix_ns = shifted_unix_ns(
server_audio_push_unix_ns,
-audio_delay_ms,
)
tethys_video_unix_ns = ( tethys_video_unix_ns = (
capture_start_unix_ns + int(round(tethys_video_time_s * 1_000_000_000.0)) capture_start_unix_ns + int(round(tethys_video_time_s * 1_000_000_000.0))
if capture_start_unix_ns is not None and tethys_video_time_s is not None if capture_start_unix_ns is not None and tethys_video_time_s is not None
@ -1593,22 +1659,38 @@ for observed in report.get("paired_events", []):
if server_audio_push_unix_ns is not None and theia_to_tethys_offset_ns is not None if server_audio_push_unix_ns is not None and theia_to_tethys_offset_ns is not None
else None else None
) )
video_freshness_ms = ( corrected_server_video_pipeline_unix_ns = (
server_video_pipeline_unix_ns + theia_to_tethys_offset_ns
if server_video_pipeline_unix_ns is not None and theia_to_tethys_offset_ns is not None
else None
)
corrected_server_audio_pipeline_unix_ns = (
server_audio_pipeline_unix_ns + theia_to_tethys_offset_ns
if server_audio_pipeline_unix_ns is not None and theia_to_tethys_offset_ns is not None
else None
)
video_sink_handoff_overhead_ms = (
(tethys_video_unix_ns - corrected_server_video_feed_unix_ns) / 1_000_000.0 (tethys_video_unix_ns - corrected_server_video_feed_unix_ns) / 1_000_000.0
if tethys_video_unix_ns is not None and corrected_server_video_feed_unix_ns is not None if tethys_video_unix_ns is not None and corrected_server_video_feed_unix_ns is not None
else None else None
) )
audio_freshness_ms = ( audio_sink_handoff_overhead_ms = (
(tethys_audio_unix_ns - corrected_server_audio_push_unix_ns) / 1_000_000.0 (tethys_audio_unix_ns - corrected_server_audio_push_unix_ns) / 1_000_000.0
if tethys_audio_unix_ns is not None and corrected_server_audio_push_unix_ns is not None if tethys_audio_unix_ns is not None and corrected_server_audio_push_unix_ns is not None
else None else None
) )
video_event_age_ms = ( video_freshness_ms = (
video_freshness_ms + video_delay_ms if video_freshness_ms is not None else None (tethys_video_unix_ns - corrected_server_video_pipeline_unix_ns) / 1_000_000.0
if tethys_video_unix_ns is not None and corrected_server_video_pipeline_unix_ns is not None
else None
) )
audio_event_age_ms = ( audio_freshness_ms = (
audio_freshness_ms + audio_delay_ms if audio_freshness_ms is not None else None (tethys_audio_unix_ns - corrected_server_audio_pipeline_unix_ns) / 1_000_000.0
if tethys_audio_unix_ns is not None and corrected_server_audio_pipeline_unix_ns is not None
else None
) )
video_event_age_ms = video_freshness_ms
audio_event_age_ms = audio_freshness_ms
residual_path_skew_ms = ( residual_path_skew_ms = (
observed_skew_ms - server_feed_delta_ms observed_skew_ms - server_feed_delta_ms
if observed_skew_ms is not None and server_feed_delta_ms is not None if observed_skew_ms is not None and server_feed_delta_ms is not None
@ -1633,12 +1715,16 @@ for observed in report.get("paired_events", []):
"server_audio_push_monotonic_us": server.get("audio_push_monotonic_us"), "server_audio_push_monotonic_us": server.get("audio_push_monotonic_us"),
"server_video_feed_unix_ns": server_video_feed_unix_ns, "server_video_feed_unix_ns": server_video_feed_unix_ns,
"server_audio_push_unix_ns": server_audio_push_unix_ns, "server_audio_push_unix_ns": server_audio_push_unix_ns,
"server_video_pipeline_unix_ns": server_video_pipeline_unix_ns,
"server_audio_pipeline_unix_ns": server_audio_pipeline_unix_ns,
"tethys_video_unix_ns": tethys_video_unix_ns, "tethys_video_unix_ns": tethys_video_unix_ns,
"tethys_audio_unix_ns": tethys_audio_unix_ns, "tethys_audio_unix_ns": tethys_audio_unix_ns,
"server_video_feed_s": server_video_feed_s, "server_video_feed_s": server_video_feed_s,
"server_audio_push_s": server_audio_push_s, "server_audio_push_s": server_audio_push_s,
"video_freshness_ms": video_freshness_ms, "video_freshness_ms": video_freshness_ms,
"audio_freshness_ms": audio_freshness_ms, "audio_freshness_ms": audio_freshness_ms,
"video_sink_handoff_overhead_ms": video_sink_handoff_overhead_ms,
"audio_sink_handoff_overhead_ms": audio_sink_handoff_overhead_ms,
"video_event_age_ms": video_event_age_ms, "video_event_age_ms": video_event_age_ms,
"audio_event_age_ms": audio_event_age_ms, "audio_event_age_ms": audio_event_age_ms,
"server_feed_delta_ms": server_feed_delta_ms, "server_feed_delta_ms": server_feed_delta_ms,
@ -1656,12 +1742,18 @@ server_model = fit_linear(joined, "server_feed_delta_ms")
residual_model = fit_linear(joined, "residual_path_skew_ms") residual_model = fit_linear(joined, "residual_path_skew_ms")
video_freshness_model = fit_linear(joined, "video_freshness_ms") video_freshness_model = fit_linear(joined, "video_freshness_ms")
audio_freshness_model = fit_linear(joined, "audio_freshness_ms") audio_freshness_model = fit_linear(joined, "audio_freshness_ms")
video_sink_handoff_model = fit_linear(joined, "video_sink_handoff_overhead_ms")
audio_sink_handoff_model = fit_linear(joined, "audio_sink_handoff_overhead_ms")
video_event_age_model = fit_linear(joined, "video_event_age_ms") video_event_age_model = fit_linear(joined, "video_event_age_ms")
audio_event_age_model = fit_linear(joined, "audio_event_age_ms") audio_event_age_model = fit_linear(joined, "audio_event_age_ms")
video_freshness_stats = stats(joined, "video_freshness_ms") video_freshness_stats = stats(joined, "video_freshness_ms")
audio_freshness_stats = stats(joined, "audio_freshness_ms") audio_freshness_stats = stats(joined, "audio_freshness_ms")
video_event_age_stats = shift_stats(video_freshness_stats, video_delay_ms) video_sink_handoff_stats = stats(joined, "video_sink_handoff_overhead_ms")
audio_event_age_stats = shift_stats(audio_freshness_stats, audio_delay_ms) audio_sink_handoff_stats = stats(joined, "audio_sink_handoff_overhead_ms")
video_event_age_stats = dict(video_freshness_stats)
audio_event_age_stats = dict(audio_freshness_stats)
video_event_age_stats["intentional_delay_ms"] = video_delay_ms
audio_event_age_stats["intentional_delay_ms"] = audio_delay_ms
server_observed_correlation = correlation(joined, "server_feed_delta_ms", "observed_skew_ms") server_observed_correlation = correlation(joined, "server_feed_delta_ms", "observed_skew_ms")
sync_verdict = report.get("verdict") or {} sync_verdict = report.get("verdict") or {}
sync_passed = sync_verdict.get("passed") is True sync_passed = sync_verdict.get("passed") is True
@ -1726,6 +1818,103 @@ event_age_min_values = [
if value is not None and math.isfinite(value) if value is not None and math.isfinite(value)
] ]
freshness_min_event_age_ms = min(event_age_min_values) if event_age_min_values else None freshness_min_event_age_ms = min(event_age_min_values) if event_age_min_values else None
smoothness = analyze_smoothness(capture_path, report, timeline)
media_timeline = smoothness.get("media_timeline") or {}
def negative_rows(rows, key, limit_ms):
return [
row
for row in rows
if row.get(key) is not None
and math.isfinite(row[key])
and row[key] < -limit_ms
]
video_negative_rows = negative_rows(joined, "video_freshness_ms", clock_uncertainty_ms)
audio_negative_rows = negative_rows(joined, "audio_freshness_ms", clock_uncertainty_ms)
capture_timebase_status = "valid"
capture_timebase_reason = "Tethys observation timestamps are monotonic against server pipeline references"
capture_timebase_reasons = []
if not joined:
capture_timebase_status = "unknown"
capture_timebase_reason = "no joined coded events are available"
elif not clock_alignment_available:
capture_timebase_status = "unknown"
capture_timebase_reason = "server/capture clock alignment is unavailable"
elif video_negative_rows or audio_negative_rows:
capture_timebase_status = "invalid"
if video_negative_rows:
capture_timebase_reasons.append(
f"video observed before server pipeline injection {len(video_negative_rows)} time(s)"
)
if audio_negative_rows:
capture_timebase_reasons.append(
f"audio observed before server pipeline injection {len(audio_negative_rows)} time(s)"
)
span_gap_s = finite(media_timeline.get("audio_video_span_gap_s"))
if span_gap_s is not None and abs(span_gap_s) > 1.0:
relation = "shorter" if span_gap_s > 0 else "longer"
capture_timebase_status = "invalid"
capture_timebase_reasons.append(
f"captured audio stream is {abs(span_gap_s):.3f}s {relation} than video; "
"the recording cannot place audio events on the video capture timeline"
)
if capture_timebase_reasons:
capture_timebase_reason = "; ".join(capture_timebase_reasons)
server_start_capture_time_s = corrected_capture_time_s_from_unix_ns(
as_int_or_none(timeline.get("server_start_unix_ns"))
)
first_timing_row = joined[0] if joined else {}
first_event_timing = {
"event_id": first_timing_row.get("event_id"),
"code": first_timing_row.get("code"),
"server_audio_pipeline_capture_s": corrected_capture_time_s_from_unix_ns(
first_timing_row.get("server_audio_pipeline_unix_ns")
),
"server_video_pipeline_capture_s": corrected_capture_time_s_from_unix_ns(
first_timing_row.get("server_video_pipeline_unix_ns")
),
"server_audio_sink_push_capture_s": corrected_capture_time_s_from_unix_ns(
first_timing_row.get("server_audio_push_unix_ns")
),
"server_video_sink_feed_capture_s": corrected_capture_time_s_from_unix_ns(
first_timing_row.get("server_video_feed_unix_ns")
),
"tethys_audio_observed_s": first_timing_row.get("tethys_audio_time_s"),
"tethys_video_observed_s": first_timing_row.get("tethys_video_time_s"),
"audio_pipeline_freshness_ms": first_timing_row.get("audio_freshness_ms"),
"video_pipeline_freshness_ms": first_timing_row.get("video_freshness_ms"),
}
timing_map = {
"schema": "lesavka.output-timing-map.v1",
"scope": "capture-relative map of server pipeline reference, sink handoff, and Tethys observation times",
"injection_scope": timeline.get("injection_scope", "server-final-output-handoff"),
"server_pipeline_reference": timeline.get(
"server_pipeline_reference",
"generated media PTS before intentional sync delay",
),
"sink_handoff_path": timeline.get(
"sink_handoff_path",
"video CameraRelay::feed; audio Voice::push",
),
"client_uplink_included": bool(timeline.get("client_uplink_included", False)),
"freshness_clock_starts_at": "server pipeline reference before intentional sync delay",
"capture_start_unix_ns": capture_start_unix_ns,
"server_start_unix_ns": as_int_or_none(timeline.get("server_start_unix_ns")),
"server_start_capture_time_s": server_start_capture_time_s,
"capture_to_server_start_ms": (
server_start_capture_time_s * 1000.0 if server_start_capture_time_s is not None else None
),
"probe_warmup_ms": as_float(timeline.get("warmup_us"), 0.0) / 1000.0,
"probe_duration_ms": as_float(timeline.get("duration_us"), 0.0) / 1000.0,
"intentional_audio_delay_ms": audio_delay_ms,
"intentional_video_delay_ms": video_delay_ms,
"first_event": first_event_timing,
}
if not event_age_p95_values: if not event_age_p95_values:
freshness_status = "unknown" freshness_status = "unknown"
freshness_reason = "clock-aligned server feed and Tethys capture timestamps were not available" freshness_reason = "clock-aligned server feed and Tethys capture timestamps were not available"
@ -1735,18 +1924,21 @@ elif not clock_alignment_available or clock_uncertainty_ms > max_clock_uncertain
f"clock uncertainty {clock_uncertainty_ms:.1f} ms exceeds " f"clock uncertainty {clock_uncertainty_ms:.1f} ms exceeds "
f"{max_clock_uncertainty_ms:.1f} ms freshness measurement limit" f"{max_clock_uncertainty_ms:.1f} ms freshness measurement limit"
) )
elif capture_timebase_status == "invalid":
freshness_status = "invalid"
freshness_reason = f"capture timebase invalid: {capture_timebase_reason}"
elif freshness_min_event_age_ms is not None and freshness_min_event_age_ms < -clock_uncertainty_ms: elif freshness_min_event_age_ms is not None and freshness_min_event_age_ms < -clock_uncertainty_ms:
freshness_status = "invalid" freshness_status = "invalid"
freshness_reason = ( freshness_reason = (
f"one media stream has impossible negative RC event age beyond clock uncertainty: " f"one media stream has impossible negative pipeline freshness beyond clock uncertainty: "
f"minimum event age {freshness_min_event_age_ms:.1f} ms, uncertainty " f"minimum freshness {freshness_min_event_age_ms:.1f} ms, uncertainty "
f"{clock_uncertainty_ms:.1f} ms" f"{clock_uncertainty_ms:.1f} ms"
) )
elif freshness_worst_event_p95_ms < -clock_uncertainty_ms: elif freshness_worst_event_p95_ms < -clock_uncertainty_ms:
freshness_status = "invalid" freshness_status = "invalid"
freshness_reason = ( freshness_reason = (
f"freshness was negative beyond clock uncertainty: " f"freshness was negative beyond clock uncertainty: "
f"worst RC event-age p95 {freshness_worst_event_p95_ms:.1f} ms, uncertainty " f"worst pipeline p95 {freshness_worst_event_p95_ms:.1f} ms, uncertainty "
f"{clock_uncertainty_ms:.1f} ms" f"{clock_uncertainty_ms:.1f} ms"
) )
elif not sync_passed: elif not sync_passed:
@ -1760,14 +1952,14 @@ elif freshness_worst_event_with_uncertainty_ms <= max_freshness_age_ms and (
): ):
freshness_status = "pass" freshness_status = "pass"
freshness_reason = ( freshness_reason = (
f"worst RC event-age p95 {freshness_worst_event_p95_ms:.1f} ms + clock uncertainty " f"worst pipeline p95 {freshness_worst_event_p95_ms:.1f} ms + clock uncertainty "
f"{clock_uncertainty_ms:.1f} ms <= {max_freshness_age_ms:.1f} ms and worst freshness drift " f"{clock_uncertainty_ms:.1f} ms <= {max_freshness_age_ms:.1f} ms and worst freshness drift "
f"{(freshness_worst_drift_ms or 0.0):.1f} ms <= {max_freshness_drift_ms:.1f} ms" f"{(freshness_worst_drift_ms or 0.0):.1f} ms <= {max_freshness_drift_ms:.1f} ms"
) )
else: else:
freshness_status = "fail" freshness_status = "fail"
freshness_reason = ( freshness_reason = (
f"worst RC event-age p95 " f"worst pipeline p95 "
f"{freshness_worst_event_p95_ms if freshness_worst_event_p95_ms is not None else 0.0:.1f} ms " f"{freshness_worst_event_p95_ms if freshness_worst_event_p95_ms is not None else 0.0:.1f} ms "
f"+ clock uncertainty {clock_uncertainty_ms:.1f} ms " f"+ clock uncertainty {clock_uncertainty_ms:.1f} ms "
f"(limit {max_freshness_age_ms:.1f} ms), worst freshness drift " f"(limit {max_freshness_age_ms:.1f} ms), worst freshness drift "
@ -1775,14 +1967,13 @@ else:
f"(limit {max_freshness_drift_ms:.1f} ms)" f"(limit {max_freshness_drift_ms:.1f} ms)"
) )
smoothness = analyze_smoothness(capture_path, report, timeline)
artifact = { artifact = {
"schema": "lesavka.output-delay-correlation.v1", "schema": "lesavka.output-delay-correlation.v1",
"report_json": report_path, "report_json": report_path,
"server_timeline_json": timeline_path, "server_timeline_json": timeline_path,
"joined_event_count": len(joined), "joined_event_count": len(joined),
"audio_after_video_positive": True, "audio_after_video_positive": True,
"timing_map": timing_map,
"observed_skew_model": observed_model, "observed_skew_model": observed_model,
"server_feed_delta_model": server_model, "server_feed_delta_model": server_model,
"residual_path_skew_model": residual_model, "residual_path_skew_model": residual_model,
@ -1790,8 +1981,15 @@ artifact = {
"schema": "lesavka.output-freshness-summary.v1", "schema": "lesavka.output-freshness-summary.v1",
"status": freshness_status, "status": freshness_status,
"reason": freshness_reason, "reason": freshness_reason,
"scope": "clock-corrected server feed to Tethys capture event from the same paired signatures", "scope": "clock-corrected server pipeline reference to Tethys observed playback from the same paired signatures",
"capture_start_unix_ns": capture_start_unix_ns, "capture_start_unix_ns": capture_start_unix_ns,
"capture_timebase": {
"status": capture_timebase_status,
"reason": capture_timebase_reason,
"valid": capture_timebase_status == "valid",
"video_observed_before_injection_count": len(video_negative_rows),
"audio_observed_before_injection_count": len(audio_negative_rows),
},
"clock_alignment": clock_alignment, "clock_alignment": clock_alignment,
"clock_uncertainty_ms": clock_uncertainty_ms, "clock_uncertainty_ms": clock_uncertainty_ms,
"max_age_limit_ms": max_freshness_age_ms, "max_age_limit_ms": max_freshness_age_ms,
@ -1799,19 +1997,35 @@ artifact = {
"max_clock_uncertainty_ms": max_clock_uncertainty_ms, "max_clock_uncertainty_ms": max_clock_uncertainty_ms,
"intentional_audio_delay_ms": audio_delay_ms, "intentional_audio_delay_ms": audio_delay_ms,
"intentional_video_delay_ms": video_delay_ms, "intentional_video_delay_ms": video_delay_ms,
"worst_p95_path_freshness_ms": freshness_worst_p95_ms, "worst_p95_sink_handoff_overhead_ms": max(
[
value
for value in [
video_sink_handoff_stats.get("p95_ms"),
audio_sink_handoff_stats.get("p95_ms"),
]
if value is not None
],
default=None,
),
"worst_p95_pipeline_freshness_ms": freshness_worst_p95_ms,
"worst_event_age_p95_ms": freshness_worst_event_p95_ms, "worst_event_age_p95_ms": freshness_worst_event_p95_ms,
"worst_event_age_with_uncertainty_ms": freshness_worst_event_with_uncertainty_ms, "worst_event_age_with_uncertainty_ms": freshness_worst_event_with_uncertainty_ms,
"minimum_event_age_ms": freshness_min_event_age_ms, "minimum_event_age_ms": freshness_min_event_age_ms,
"minimum_pipeline_freshness_ms": freshness_min_event_age_ms,
"sync_passed": sync_passed, "sync_passed": sync_passed,
"worst_p95_freshness_ms": freshness_worst_event_p95_ms, "worst_p95_freshness_ms": freshness_worst_event_p95_ms,
"worst_freshness_drift_ms": freshness_worst_drift_ms, "worst_freshness_drift_ms": freshness_worst_drift_ms,
"video_freshness_stats": video_freshness_stats, "video_freshness_stats": video_freshness_stats,
"audio_freshness_stats": audio_freshness_stats, "audio_freshness_stats": audio_freshness_stats,
"video_sink_handoff_overhead_stats": video_sink_handoff_stats,
"audio_sink_handoff_overhead_stats": audio_sink_handoff_stats,
"video_event_age_stats": video_event_age_stats, "video_event_age_stats": video_event_age_stats,
"audio_event_age_stats": audio_event_age_stats, "audio_event_age_stats": audio_event_age_stats,
"video_freshness_model": video_freshness_model, "video_freshness_model": video_freshness_model,
"audio_freshness_model": audio_freshness_model, "audio_freshness_model": audio_freshness_model,
"video_sink_handoff_overhead_model": video_sink_handoff_model,
"audio_sink_handoff_overhead_model": audio_sink_handoff_model,
"video_event_age_model": video_event_age_model, "video_event_age_model": video_event_age_model,
"audio_event_age_model": audio_event_age_model, "audio_event_age_model": audio_event_age_model,
}, },
@ -1854,10 +2068,14 @@ with pathlib.Path(output_csv_path).open("w", newline="", encoding="utf-8") as ha
"server_audio_push_s", "server_audio_push_s",
"server_video_feed_unix_ns", "server_video_feed_unix_ns",
"server_audio_push_unix_ns", "server_audio_push_unix_ns",
"server_video_pipeline_unix_ns",
"server_audio_pipeline_unix_ns",
"tethys_video_unix_ns", "tethys_video_unix_ns",
"tethys_audio_unix_ns", "tethys_audio_unix_ns",
"video_freshness_ms", "video_freshness_ms",
"audio_freshness_ms", "audio_freshness_ms",
"video_sink_handoff_overhead_ms",
"audio_sink_handoff_overhead_ms",
"video_event_age_ms", "video_event_age_ms",
"audio_event_age_ms", "audio_event_age_ms",
"server_feed_delta_ms", "server_feed_delta_ms",
@ -1871,6 +2089,19 @@ with pathlib.Path(output_csv_path).open("w", newline="", encoding="utf-8") as ha
for row in joined: for row in joined:
writer.writerow({key: row.get(key) for key in fieldnames}) writer.writerow({key: row.get(key) for key in fieldnames})
def fmt_number(value, unit="", precision=1):
if value is None:
return "n/a"
try:
value = float(value)
except Exception:
return "n/a"
if not math.isfinite(value):
return "n/a"
return f"{value:.{precision}f}{unit}"
lines = [ lines = [
f"Output-delay correlation for {report_path}", f"Output-delay correlation for {report_path}",
f"- joined events: {len(joined)}", f"- joined events: {len(joined)}",
@ -1882,19 +2113,32 @@ lines = [
f"- server/observed correlation: {server_observed_correlation:+.3f}", f"- server/observed correlation: {server_observed_correlation:+.3f}",
f"- server drift share of observed: {server_share:.3f}", f"- server drift share of observed: {server_share:.3f}",
"", "",
"Timing map",
"- scope: capture-relative server pipeline reference, sink handoff, and Tethys observation",
f"- injection point: {timing_map.get('injection_scope')} ({timing_map.get('sink_handoff_path')}); client uplink included={timing_map.get('client_uplink_included')}",
f"- freshness clock starts at: {timing_map.get('freshness_clock_starts_at')}",
f"- capture to server probe start: {fmt_number(timing_map.get('capture_to_server_start_ms'), ' ms')} (setup/pre-roll, not media latency)",
f"- probe warmup before first coded event: {fmt_number(timing_map.get('probe_warmup_ms'), ' ms')}",
f"- first event server pipeline refs: audio {fmt_number(first_event_timing.get('server_audio_pipeline_capture_s'), 's', 3)}, video {fmt_number(first_event_timing.get('server_video_pipeline_capture_s'), 's', 3)}",
f"- first event sink handoff: audio {fmt_number(first_event_timing.get('server_audio_sink_push_capture_s'), 's', 3)}, video {fmt_number(first_event_timing.get('server_video_sink_feed_capture_s'), 's', 3)}",
f"- first event Tethys observation: audio {fmt_number(first_event_timing.get('tethys_audio_observed_s'), 's', 3)}, video {fmt_number(first_event_timing.get('tethys_video_observed_s'), 's', 3)}",
f"- first event pipeline freshness: audio {fmt_number(first_event_timing.get('audio_pipeline_freshness_ms'), ' ms')}, video {fmt_number(first_event_timing.get('video_pipeline_freshness_ms'), ' ms')}",
"",
f"Output freshness for {report_path}", f"Output freshness for {report_path}",
"- scope: clock-corrected server feed to Tethys capture event from the same paired signatures", "- scope: clock-corrected server pipeline reference to Tethys observed playback from the same paired signatures",
f"- freshness status: {freshness_status} ({freshness_reason})", f"- freshness status: {freshness_status} ({freshness_reason})",
f"- capture timebase: {capture_timebase_status} ({capture_timebase_reason})",
f"- clock uncertainty: +/-{clock_uncertainty_ms:.1f} ms", f"- clock uncertainty: +/-{clock_uncertainty_ms:.1f} ms",
f"- intentional sync delays: audio {audio_delay_ms:+.1f} ms, video {video_delay_ms:+.1f} ms", f"- intentional sync delays: audio {audio_delay_ms:+.1f} ms, video {video_delay_ms:+.1f} ms",
f"- media timestamp path offset: video median {video_freshness_stats.get('median_ms') or 0.0:.1f} ms / p95 {video_freshness_stats.get('p95_ms') or 0.0:.1f} ms / max {video_freshness_stats.get('max_ms') or 0.0:.1f} ms; audio median {audio_freshness_stats.get('median_ms') or 0.0:.1f} ms / p95 {audio_freshness_stats.get('p95_ms') or 0.0:.1f} ms / max {audio_freshness_stats.get('max_ms') or 0.0:.1f} ms", f"- sink handoff overhead: video median {fmt_number(video_sink_handoff_stats.get('median_ms'), ' ms')} / p95 {fmt_number(video_sink_handoff_stats.get('p95_ms'), ' ms')} / max {fmt_number(video_sink_handoff_stats.get('max_ms'), ' ms')}; audio median {fmt_number(audio_sink_handoff_stats.get('median_ms'), ' ms')} / p95 {fmt_number(audio_sink_handoff_stats.get('p95_ms'), ' ms')} / max {fmt_number(audio_sink_handoff_stats.get('max_ms'), ' ms')}",
f"- RC target event age: video median {video_event_age_stats.get('median_ms') or 0.0:.1f} ms / p95 {video_event_age_stats.get('p95_ms') or 0.0:.1f} ms / max {video_event_age_stats.get('max_ms') or 0.0:.1f} ms; audio median {audio_event_age_stats.get('median_ms') or 0.0:.1f} ms / p95 {audio_event_age_stats.get('p95_ms') or 0.0:.1f} ms / max {audio_event_age_stats.get('max_ms') or 0.0:.1f} ms", f"- pipeline freshness: video median {fmt_number(video_freshness_stats.get('median_ms'), ' ms')} / p95 {fmt_number(video_freshness_stats.get('p95_ms'), ' ms')} / max {fmt_number(video_freshness_stats.get('max_ms'), ' ms')}; audio median {fmt_number(audio_freshness_stats.get('median_ms'), ' ms')} / p95 {fmt_number(audio_freshness_stats.get('p95_ms'), ' ms')} / max {fmt_number(audio_freshness_stats.get('max_ms'), ' ms')}",
f"- freshness budget: worst RC event-age p95 {(freshness_worst_event_p95_ms or 0.0):.1f} ms + clock uncertainty {clock_uncertainty_ms:.1f} ms = {(freshness_worst_event_with_uncertainty_ms or 0.0):.1f} ms vs limit {max_freshness_age_ms:.1f} ms", f"- freshness budget: worst pipeline p95 {fmt_number(freshness_worst_event_p95_ms, ' ms')} + clock uncertainty {clock_uncertainty_ms:.1f} ms = {fmt_number(freshness_worst_event_with_uncertainty_ms, ' ms')} vs limit {max_freshness_age_ms:.1f} ms",
f"- video event-age drift: {video_event_age_model.get('drift_ms', 0.0):+.1f} ms over paired events ({video_event_age_model.get('slope_ms_per_s', 0.0):+.3f} ms/s)", f"- video freshness drift: {video_event_age_model.get('drift_ms', 0.0):+.1f} ms over paired events ({video_event_age_model.get('slope_ms_per_s', 0.0):+.3f} ms/s)",
f"- audio event-age drift: {audio_event_age_model.get('drift_ms', 0.0):+.1f} ms over paired events ({audio_event_age_model.get('slope_ms_per_s', 0.0):+.3f} ms/s)", f"- audio freshness drift: {audio_event_age_model.get('drift_ms', 0.0):+.1f} ms over paired events ({audio_event_age_model.get('slope_ms_per_s', 0.0):+.3f} ms/s)",
"", "",
"Output smoothness", "Output smoothness",
f"- window: {smoothness.get('window_start_s') or 0.0:.3f}s to {smoothness.get('window_end_s') or 0.0:.3f}s", f"- window: {smoothness.get('window_start_s') or 0.0:.3f}s to {smoothness.get('window_end_s') or 0.0:.3f}s",
f"- capture media timeline: video span {fmt_number(media_timeline.get('video_span_s'), 's', 3)}, audio span {fmt_number(media_timeline.get('audio_span_s'), 's', 3)}, audio/video span gap {fmt_number(media_timeline.get('audio_video_span_gap_s'), 's', 3)}",
f"- video cadence: frames {smoothness.get('video', {}).get('timestamps', 0)}, expected interval {(smoothness.get('video', {}).get('expected_interval_ms') or 0.0):.1f} ms, p95 jitter {(smoothness.get('video', {}).get('jitter_stats', {}).get('p95_jitter_ms') or 0.0):.1f} ms, max interval {(smoothness.get('video', {}).get('interval_stats', {}).get('max_interval_ms') or 0.0):.1f} ms, hiccups {smoothness.get('video', {}).get('hiccup_count', 0)}", f"- video cadence: frames {smoothness.get('video', {}).get('timestamps', 0)}, expected interval {(smoothness.get('video', {}).get('expected_interval_ms') or 0.0):.1f} ms, p95 jitter {(smoothness.get('video', {}).get('jitter_stats', {}).get('p95_jitter_ms') or 0.0):.1f} ms, max interval {(smoothness.get('video', {}).get('interval_stats', {}).get('max_interval_ms') or 0.0):.1f} ms, hiccups {smoothness.get('video', {}).get('hiccup_count', 0)}",
f"- video continuity: decoded {smoothness.get('video', {}).get('decoded_frames', 0)}, duplicates {smoothness.get('video', {}).get('duplicate_frames', 0)}, estimated missing {smoothness.get('video', {}).get('estimated_missing_frames', 0)}, undecodable {smoothness.get('video', {}).get('undecodable_frames', 0)}", f"- video continuity: decoded {smoothness.get('video', {}).get('decoded_frames', 0)}, duplicates {smoothness.get('video', {}).get('duplicate_frames', 0)}, estimated missing {smoothness.get('video', {}).get('estimated_missing_frames', 0)}, undecodable {smoothness.get('video', {}).get('undecodable_frames', 0)}",
f"- audio packet cadence: packets {smoothness.get('audio', {}).get('packet_cadence', {}).get('timestamps', 0)}, p95 jitter {(smoothness.get('audio', {}).get('packet_cadence', {}).get('jitter_stats', {}).get('p95_jitter_ms') or 0.0):.1f} ms, max interval {(smoothness.get('audio', {}).get('packet_cadence', {}).get('interval_stats', {}).get('max_interval_ms') or 0.0):.1f} ms, hiccups {smoothness.get('audio', {}).get('packet_cadence', {}).get('hiccup_count', 0)}", f"- audio packet cadence: packets {smoothness.get('audio', {}).get('packet_cadence', {}).get('timestamps', 0)}, p95 jitter {(smoothness.get('audio', {}).get('packet_cadence', {}).get('jitter_stats', {}).get('p95_jitter_ms') or 0.0):.1f} ms, max interval {(smoothness.get('audio', {}).get('packet_cadence', {}).get('interval_stats', {}).get('max_interval_ms') or 0.0):.1f} ms, hiccups {smoothness.get('audio', {}).get('packet_cadence', {}).get('hiccup_count', 0)}",
@ -2081,6 +2325,7 @@ ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
"${REMOTE_CAPTURE_STACK}" \ "${REMOTE_CAPTURE_STACK}" \
"${REMOTE_PULSE_CAPTURE_TOOL}" \ "${REMOTE_PULSE_CAPTURE_TOOL}" \
"${REMOTE_PULSE_VIDEO_MODE}" \ "${REMOTE_PULSE_VIDEO_MODE}" \
"${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE}" \
"${REMOTE_AUDIO_SOURCE}" \ "${REMOTE_AUDIO_SOURCE}" \
"${REMOTE_AUDIO_QUIESCE_USER_AUDIO}" \ "${REMOTE_AUDIO_QUIESCE_USER_AUDIO}" \
"${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" \ "${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" \
@ -2098,11 +2343,12 @@ video_format=$6
remote_capture_stack=$7 remote_capture_stack=$7
remote_pulse_capture_tool=$8 remote_pulse_capture_tool=$8
remote_pulse_video_mode=$9 remote_pulse_video_mode=$9
remote_audio_source=${10} remote_pulse_audio_anchor_silence=${10}
remote_audio_quiesce_user_audio=${11} remote_audio_source=${11}
remote_capture_allow_alsa_fallback=${12} remote_audio_quiesce_user_audio=${12}
remote_capture_preroll_discard_seconds=${13} remote_capture_allow_alsa_fallback=${13}
remote_capture_ready_settle_seconds=${14} remote_capture_preroll_discard_seconds=${14}
remote_capture_ready_settle_seconds=${15}
rm -f "${remote_capture}" rm -f "${remote_capture}"
@ -2234,6 +2480,14 @@ gst_video_decode_chain() {
esac esac
} }
gst_audio_mixer_element() {
if gst-inspect-1.0 audiomixer 2>/dev/null | grep -q 'ignore-inactive-pads'; then
printf 'audiomixer name=amix ignore-inactive-pads=true'
else
printf 'audiomixer name=amix'
fi
}
current_video_profile() { current_video_profile() {
if ! command -v v4l2-ctl >/dev/null 2>&1; then if ! command -v v4l2-ctl >/dev/null 2>&1; then
return 1 return 1
@ -2443,6 +2697,7 @@ if [[ -n "${video_format}" ]]; then
fi fi
gst_source_caps="$(gst_video_source_caps)" gst_source_caps="$(gst_video_source_caps)"
gst_decode_chain="$(gst_video_decode_chain)" gst_decode_chain="$(gst_video_decode_chain)"
gst_audio_mixer="$(gst_audio_mixer_element)"
run_ffmpeg_capture() { run_ffmpeg_capture() {
local rc=0 local rc=0
@ -2618,40 +2873,85 @@ elif [[ "${capture_mode}" == "pulse" ]]; then
;; ;;
esac esac
;; ;;
gst) gst)
case "${remote_pulse_video_mode}" in if [[ "${remote_pulse_audio_anchor_silence}" == "1" ]]; then
copy) printf 'anchoring Pulse capture audio timeline with generated silence\n' >&2
if [[ "${video_format}" != "mjpeg" && "${video_format}" != "MJPG" && -n "${video_format}" ]]; then fi
printf 'gst copy mode only supports mjpeg input, got %s\n' "${video_format}" >&2 case "${remote_pulse_video_mode}" in
exit 64 copy)
fi if [[ "${video_format}" != "mjpeg" && "${video_format}" != "MJPG" && -n "${video_format}" ]]; then
run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \ printf 'gst copy mode only supports mjpeg input, got %s\n' "${video_format}" >&2
gst-launch-1.0 -q -e \ exit 64
matroskamux name=mux ! filesink location="${remote_capture}" \ fi
v4l2src device="${resolved_video_device}" do-timestamp=true ! \ if [[ "${remote_pulse_audio_anchor_silence}" == "1" ]]; then
${gst_source_caps} ! \ run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \
queue ! mux. \ gst-launch-1.0 -q -e \
pulsesrc device="${pulse_source}" do-timestamp=true ! \ matroskamux name=mux ! filesink location="${remote_capture}" \
audio/x-raw,rate=48000,channels=2 ! \ v4l2src device="${resolved_video_device}" do-timestamp=true ! \
audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \ ${gst_source_caps} ! \
queue ! mux. queue ! mux. \
;; ${gst_audio_mixer} ! \
cfr) audio/x-raw,rate=48000,channels=2 ! \
run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \ queue ! mux. \
gst-launch-1.0 -q -e \ audiotestsrc wave=silence is-live=true do-timestamp=true ! \
matroskamux name=mux ! filesink location="${remote_capture}" \ audio/x-raw,rate=48000,channels=2 ! \
v4l2src device="${resolved_video_device}" do-timestamp=true ! \ queue ! amix. \
${gst_source_caps} ! \ pulsesrc device="${pulse_source}" do-timestamp=true ! \
${gst_decode_chain} \ audio/x-raw,rate=48000,channels=2 ! \
videoconvert ! videorate ! video/x-raw,framerate="${resolved_video_fps}"/1 ! \ audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \
x264enc tune=zerolatency speed-preset=ultrafast key-int-max=1 bitrate=5000 ! \ queue ! amix.
h264parse ! \ else
queue ! mux. \ run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \
pulsesrc device="${pulse_source}" do-timestamp=true ! \ gst-launch-1.0 -q -e \
audio/x-raw,rate=48000,channels=2 ! \ matroskamux name=mux ! filesink location="${remote_capture}" \
audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \ v4l2src device="${resolved_video_device}" do-timestamp=true ! \
queue ! mux. ${gst_source_caps} ! \
;; queue ! mux. \
pulsesrc device="${pulse_source}" do-timestamp=true ! \
audio/x-raw,rate=48000,channels=2 ! \
audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \
queue ! mux.
fi
;;
cfr)
if [[ "${remote_pulse_audio_anchor_silence}" == "1" ]]; then
run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \
gst-launch-1.0 -q -e \
matroskamux name=mux ! filesink location="${remote_capture}" \
v4l2src device="${resolved_video_device}" do-timestamp=true ! \
${gst_source_caps} ! \
${gst_decode_chain} \
videoconvert ! videorate ! video/x-raw,framerate="${resolved_video_fps}"/1 ! \
x264enc tune=zerolatency speed-preset=ultrafast key-int-max=1 bitrate=5000 ! \
h264parse ! \
queue ! mux. \
${gst_audio_mixer} ! \
audio/x-raw,rate=48000,channels=2 ! \
queue ! mux. \
audiotestsrc wave=silence is-live=true do-timestamp=true ! \
audio/x-raw,rate=48000,channels=2 ! \
queue ! amix. \
pulsesrc device="${pulse_source}" do-timestamp=true ! \
audio/x-raw,rate=48000,channels=2 ! \
audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \
queue ! amix.
else
run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \
gst-launch-1.0 -q -e \
matroskamux name=mux ! filesink location="${remote_capture}" \
v4l2src device="${resolved_video_device}" do-timestamp=true ! \
${gst_source_caps} ! \
${gst_decode_chain} \
videoconvert ! videorate ! video/x-raw,framerate="${resolved_video_fps}"/1 ! \
x264enc tune=zerolatency speed-preset=ultrafast key-int-max=1 bitrate=5000 ! \
h264parse ! \
queue ! mux. \
pulsesrc device="${pulse_source}" do-timestamp=true ! \
audio/x-raw,rate=48000,channels=2 ! \
audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \
queue ! mux.
fi
;;
*) *)
printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2 printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2
exit 64 exit 64

View File

@ -10,7 +10,7 @@ bench = false
[package] [package]
name = "lesavka_server" name = "lesavka_server"
version = "0.19.21" version = "0.19.22"
edition = "2024" edition = "2024"
autobins = false autobins = false

View File

@ -152,6 +152,10 @@ struct OutputDelayProbeTimeline {
schema: &'static str, schema: &'static str,
origin: &'static str, origin: &'static str,
media_path: &'static str, media_path: &'static str,
injection_scope: &'static str,
server_pipeline_reference: &'static str,
sink_handoff_path: &'static str,
client_uplink_included: bool,
camera_width: u32, camera_width: u32,
camera_height: u32, camera_height: u32,
camera_fps: u32, camera_fps: u32,
@ -208,6 +212,10 @@ impl OutputDelayProbeTimeline {
schema: "lesavka.output-delay-server-timeline.v1", schema: "lesavka.output-delay-server-timeline.v1",
origin: "theia-server-generated", origin: "theia-server-generated",
media_path: "server generator -> UVC/UAC sinks", media_path: "server generator -> UVC/UAC sinks",
injection_scope: "server-final-output-handoff",
server_pipeline_reference: "generated media PTS before intentional sync delay",
sink_handoff_path: "video CameraRelay::feed; audio Voice::push",
client_uplink_included: false,
camera_width: camera.width, camera_width: camera.width,
camera_height: camera.height, camera_height: camera.height,
camera_fps: camera.fps, camera_fps: camera.fps,
@ -388,8 +396,9 @@ fn parse_event_width_codes(raw: &str) -> Result<Vec<u32>> {
/// Inputs: the active camera relay, active UAC voice sink, camera profile, and /// Inputs: the active camera relay, active UAC voice sink, camera profile, and
/// probe request timing. /// probe request timing.
/// Outputs: a small count summary after the last generated packet. /// Outputs: a small count summary after the last generated packet.
/// Why: this probe intentionally bypasses client capture/uplink so the measured /// Why: this probe intentionally bypasses client capture/uplink but uses the
/// skew is the static output-path difference between server UVC and UAC. /// same final server output handoff calls as received client media, so the
/// measured skew/freshness is the server final-handoff-to-RCT path.
#[cfg(not(coverage))] #[cfg(not(coverage))]
pub async fn run_server_output_delay_probe( pub async fn run_server_output_delay_probe(
relay: Arc<CameraRelay>, relay: Arc<CameraRelay>,
@ -797,11 +806,12 @@ fn duration_mul(duration: Duration, count: u64) -> Duration {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{ use super::{
DARK_FRAME_RGB, OutputDelayProbeTimeline, ProbeConfig, duration_us, probe_color_for_code, DARK_FRAME_RGB, EVENT_COLORS, EVENT_FREQUENCIES_HZ, OutputDelayProbeTimeline, ProbeConfig,
render_audio_chunk, unix_ns_from_start, duration_us, probe_color_for_code, render_audio_chunk, unix_ns_from_start,
}; };
use crate::camera::{CameraCodec, CameraConfig, CameraOutput}; use crate::camera::{CameraCodec, CameraConfig, CameraOutput};
use lesavka_common::lesavka::OutputDelayProbeRequest; use lesavka_common::lesavka::OutputDelayProbeRequest;
use std::collections::BTreeSet;
use std::time::Duration; use std::time::Duration;
#[test] #[test]
@ -826,6 +836,24 @@ mod tests {
assert!(ProbeConfig::from_request(&request).is_err()); assert!(ProbeConfig::from_request(&request).is_err());
} }
#[test]
fn default_probe_signatures_are_unique_for_all_coded_pairs() {
assert_eq!(EVENT_COLORS.len(), 16);
assert_eq!(EVENT_FREQUENCIES_HZ.len(), 16);
let colors = EVENT_COLORS
.iter()
.map(|color| (color.r, color.g, color.b))
.collect::<BTreeSet<_>>();
let frequencies = EVENT_FREQUENCIES_HZ
.iter()
.map(|frequency| (*frequency * 10.0).round() as i64)
.collect::<BTreeSet<_>>();
assert_eq!(colors.len(), EVENT_COLORS.len());
assert_eq!(frequencies.len(), EVENT_FREQUENCIES_HZ.len());
}
#[test] #[test]
fn audio_chunk_contains_tone_only_during_coded_pulse() { fn audio_chunk_contains_tone_only_during_coded_pulse() {
let config = ProbeConfig::from_request(&OutputDelayProbeRequest { let config = ProbeConfig::from_request(&OutputDelayProbeRequest {
@ -918,6 +946,15 @@ mod tests {
); );
let json = serde_json::to_value(timeline).expect("timeline json"); let json = serde_json::to_value(timeline).expect("timeline json");
assert_eq!(
json["injection_scope"].as_str(),
Some("server-final-output-handoff")
);
assert_eq!(
json["sink_handoff_path"].as_str(),
Some("video CameraRelay::feed; audio Voice::push")
);
assert_eq!(json["client_uplink_included"].as_bool(), Some(false));
assert_eq!( assert_eq!(
json["server_start_unix_ns"].as_u64(), json["server_start_unix_ns"].as_u64(),
Some(start_unix_ns as u64) Some(start_unix_ns as u64)

View File

@ -58,7 +58,7 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}", "LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}",
"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS:-0}", "REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS:-0}",
"REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS:-1}", "REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS:-1}",
"remote_capture_ready_settle_seconds=${14}", "remote_capture_ready_settle_seconds=${15}",
"announce_capture_start", "announce_capture_start",
"signal_capture_ready", "signal_capture_ready",
"run_tolerant_capture", "run_tolerant_capture",
@ -95,16 +95,29 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
"schema\": \"lesavka.output-delay-correlation.v1\"", "schema\": \"lesavka.output-delay-correlation.v1\"",
"schema\": \"lesavka.clock-alignment.v1\"", "schema\": \"lesavka.clock-alignment.v1\"",
"schema\": \"lesavka.output-freshness-summary.v1\"", "schema\": \"lesavka.output-freshness-summary.v1\"",
"schema\": \"lesavka.output-timing-map.v1\"",
"server_timeline_json=", "server_timeline_json=",
"clock_alignment_json: ${LOCAL_CLOCK_ALIGNMENT_JSON}", "clock_alignment_json: ${LOCAL_CLOCK_ALIGNMENT_JSON}",
"dominant_layer", "dominant_layer",
"Timing map",
"capture to server probe start",
"setup/pre-roll, not media latency",
"injection point",
"server-final-output-handoff",
"video CameraRelay::feed; audio Voice::push",
"client uplink included",
"freshness clock starts",
"freshness status", "freshness status",
"clock-corrected server feed to Tethys capture event", "clock-corrected server pipeline reference to Tethys observed playback",
"capture timebase",
"capture timebase invalid",
"intentional sync delays", "intentional sync delays",
"media timestamp path offset", "sink handoff overhead",
"RC target event age", "pipeline freshness",
"freshness budget", "freshness budget",
"Output smoothness", "Output smoothness",
"capture media timeline",
"media_timeline",
"video continuity", "video continuity",
"audio pilot continuity", "audio pilot continuity",
"schema\": \"lesavka.output-smoothness-summary.v1\"", "schema\": \"lesavka.output-smoothness-summary.v1\"",
@ -114,16 +127,20 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
"audio_event_age_ms", "audio_event_age_ms",
"intentional_video_delay_ms", "intentional_video_delay_ms",
"video_event_age_stats", "video_event_age_stats",
"video_sink_handoff_overhead_stats",
"minimum_pipeline_freshness_ms",
"worst_event_age_with_uncertainty_ms", "worst_event_age_with_uncertainty_ms",
"minimum_event_age_ms", "minimum_event_age_ms",
"sync_passed", "sync_passed",
"sync did not pass, so freshness from paired signatures is not trustworthy", "sync did not pass, so freshness from paired signatures is not trustworthy",
"impossible negative RC event age", "impossible negative pipeline freshness",
"video_delay_function_candidate", "video_delay_function_candidate",
"source\": \"direct-uvc-uac-output-probe\"", "source\": \"direct-uvc-uac-output-probe\"",
"scope\": \"server-output-static-baseline\"", "scope\": \"server-output-static-baseline\"",
"applies_to\": \"server UVC/UAC gadget output path\"", "applies_to\": \"server UVC/UAC gadget output path\"",
"measurement_host_role\": \"lab-attached USB host\"", "measurement_host_role\": \"lab-attached USB host\"",
"injection_scope\": \"server-final-output-handoff\"",
"client_uplink_included\": False",
"probe_media_origin\": \"server-generated\"", "probe_media_origin\": \"server-generated\"",
"probe_media_path\": \"server generated signatures -> UVC/UAC sinks -> lab host capture\"", "probe_media_path\": \"server generated signatures -> UVC/UAC sinks -> lab host capture\"",
"audio_after_video_positive", "audio_after_video_positive",
@ -151,6 +168,10 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
"\"${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}\"", "\"${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}\"",
"REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}", "REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}",
"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}", "REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}",
"REMOTE_PULSE_AUDIO_ANCHOR_SILENCE=${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE:-1}",
"anchoring Pulse capture audio timeline with generated silence",
"audiotestsrc wave=silence is-live=true do-timestamp=true",
"audiomixer name=amix ignore-inactive-pads=true",
"output_delay_calibration_json", "output_delay_calibration_json",
"direct UVC/UAC output-delay calibration", "direct UVC/UAC output-delay calibration",
"calibration-save-default", "calibration-save-default",
@ -288,6 +309,9 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() {
"calibration_video_target_offset_us", "calibration_video_target_offset_us",
"calibration_audio_target_offset_us", "calibration_audio_target_offset_us",
"calibration:", "calibration:",
"capture_timebase_status",
"capture timebase invalid",
"capture timing:",
"signature_coverage", "signature_coverage",
"paired coded signatures", "paired coded signatures",
"signature_missing_codes", "signature_missing_codes",