test(server-rc): harden output freshness proof
This commit is contained in:
parent
1e343057ac
commit
b4587970b4
6
Cargo.lock
generated
6
Cargo.lock
generated
@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
|
||||
|
||||
[[package]]
|
||||
name = "lesavka_client"
|
||||
version = "0.19.21"
|
||||
version = "0.19.22"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-stream",
|
||||
@ -1686,7 +1686,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lesavka_common"
|
||||
version = "0.19.21"
|
||||
version = "0.19.22"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
@ -1698,7 +1698,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lesavka_server"
|
||||
version = "0.19.21"
|
||||
version = "0.19.22"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
|
||||
@ -4,7 +4,7 @@ path = "src/main.rs"
|
||||
|
||||
[package]
|
||||
name = "lesavka_client"
|
||||
version = "0.19.21"
|
||||
version = "0.19.22"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lesavka_common"
|
||||
version = "0.19.21"
|
||||
version = "0.19.22"
|
||||
edition = "2024"
|
||||
build = "build.rs"
|
||||
|
||||
|
||||
@ -910,7 +910,9 @@ artifact_dir = pathlib.Path(artifact_dir_raw) if artifact_dir_raw else pathlib.P
|
||||
report = load_json(artifact_dir / "report.json")
|
||||
correlation = load_json(artifact_dir / "output-delay-correlation.json")
|
||||
calibration = load_json(artifact_dir / "output-delay-calibration.json")
|
||||
timing_map = correlation.get("timing_map") or {}
|
||||
freshness = correlation.get("freshness") or {}
|
||||
capture_timebase = freshness.get("capture_timebase") or {}
|
||||
smoothness = correlation.get("smoothness") or {}
|
||||
video = smoothness.get("video") or {}
|
||||
audio = smoothness.get("audio") or {}
|
||||
@ -972,6 +974,11 @@ if as_bool(require_sync_raw) and not sync_pass:
|
||||
reasons.append(f"sync did not pass: {verdict.get('status', 'unknown')}")
|
||||
if as_bool(require_freshness_raw) and not freshness_pass:
|
||||
reasons.append(f"freshness did not pass: {freshness_status}")
|
||||
if capture_timebase and capture_timebase.get("valid") is False:
|
||||
reasons.append(
|
||||
"capture timebase invalid: "
|
||||
f"{capture_timebase.get('reason', capture_timebase.get('status', 'invalid'))}"
|
||||
)
|
||||
if signature_expected > 0:
|
||||
if signature_paired < signature_expected:
|
||||
reasons.append(f"paired coded signatures {signature_paired} < expected {signature_expected}")
|
||||
@ -1010,6 +1017,7 @@ artifact = {
|
||||
"report_json": str(artifact_dir / "report.json"),
|
||||
"correlation_json": str(artifact_dir / "output-delay-correlation.json"),
|
||||
"calibration_json": str(artifact_dir / "output-delay-calibration.json"),
|
||||
"timing_map": timing_map,
|
||||
"passed": not reasons,
|
||||
"failure_reasons": reasons,
|
||||
"sync": {
|
||||
@ -1037,10 +1045,15 @@ artifact = {
|
||||
"status": freshness_status,
|
||||
"reason": freshness.get("reason", ""),
|
||||
"worst_event_age_p95_ms": freshness.get("worst_event_age_p95_ms"),
|
||||
"worst_p95_pipeline_freshness_ms": freshness.get("worst_p95_pipeline_freshness_ms"),
|
||||
"minimum_pipeline_freshness_ms": freshness.get("minimum_pipeline_freshness_ms"),
|
||||
"clock_uncertainty_ms": freshness.get("clock_uncertainty_ms"),
|
||||
"worst_event_age_with_uncertainty_ms": freshness.get("worst_event_age_with_uncertainty_ms"),
|
||||
"worst_freshness_drift_ms": freshness.get("worst_freshness_drift_ms"),
|
||||
"max_age_limit_ms": freshness.get("max_age_limit_ms"),
|
||||
"capture_timebase_status": capture_timebase.get("status"),
|
||||
"capture_timebase_valid": capture_timebase.get("valid"),
|
||||
"capture_timebase_reason": capture_timebase.get("reason"),
|
||||
},
|
||||
"smoothness": {
|
||||
"video_frames": as_int(video.get("timestamps"), 0),
|
||||
@ -1221,6 +1234,7 @@ fieldnames = [
|
||||
"freshness_status",
|
||||
"freshness_budget_ms",
|
||||
"freshness_drift_ms",
|
||||
"capture_timebase_status",
|
||||
"video_hiccups",
|
||||
"video_missing",
|
||||
"video_undecodable",
|
||||
@ -1252,6 +1266,7 @@ with summary_csv.open("w", newline="", encoding="utf-8") as handle:
|
||||
"freshness_status": (result.get("freshness") or {}).get("status"),
|
||||
"freshness_budget_ms": (result.get("freshness") or {}).get("worst_event_age_with_uncertainty_ms"),
|
||||
"freshness_drift_ms": (result.get("freshness") or {}).get("worst_freshness_drift_ms"),
|
||||
"capture_timebase_status": (result.get("freshness") or {}).get("capture_timebase_status"),
|
||||
"video_hiccups": (result.get("smoothness") or {}).get("video_hiccups"),
|
||||
"video_missing": (result.get("smoothness") or {}).get("video_estimated_missing_frames"),
|
||||
"video_undecodable": (result.get("smoothness") or {}).get("video_undecodable_frames"),
|
||||
@ -1301,6 +1316,13 @@ for result in results:
|
||||
f"pair_conf_median={sync.get('paired_confidence_median', 0.0):.3f} "
|
||||
f"raw_pair_disagreement={sync.get('activity_pair_disagreement_ms', 0.0):+.1f}ms"
|
||||
)
|
||||
if freshness.get("capture_timebase_status"):
|
||||
lines.append(
|
||||
" capture timing: "
|
||||
f"{freshness.get('capture_timebase_status')} "
|
||||
f"valid={freshness.get('capture_timebase_valid')} "
|
||||
f"reason={freshness.get('capture_timebase_reason', '')}"
|
||||
)
|
||||
if "seed_video_delay_us" in result or "seed_audio_delay_us" in result:
|
||||
lines.append(
|
||||
" tuning: "
|
||||
|
||||
@ -40,6 +40,7 @@ VIDEO_FORMAT=${VIDEO_FORMAT:-mjpeg}
|
||||
REMOTE_CAPTURE_STACK=${REMOTE_CAPTURE_STACK:-pulse}
|
||||
REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}
|
||||
REMOTE_PULSE_VIDEO_MODE=${REMOTE_PULSE_VIDEO_MODE:-copy}
|
||||
REMOTE_PULSE_AUDIO_ANCHOR_SILENCE=${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE:-1}
|
||||
REMOTE_AUDIO_SOURCE=${REMOTE_AUDIO_SOURCE:-auto}
|
||||
REMOTE_AUDIO_QUIESCE_USER_AUDIO=${REMOTE_AUDIO_QUIESCE_USER_AUDIO:-auto}
|
||||
REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}
|
||||
@ -729,6 +730,12 @@ def env_line(key, value):
|
||||
|
||||
report = json.loads(pathlib.Path(report_path).read_text())
|
||||
verdict = report.get("verdict") or {}
|
||||
try:
|
||||
correlation = json.loads(pathlib.Path(correlation_path).read_text())
|
||||
except Exception:
|
||||
correlation = {}
|
||||
freshness = correlation.get("freshness") or {}
|
||||
capture_timebase = freshness.get("capture_timebase") or {}
|
||||
|
||||
target = target.strip().lower()
|
||||
apply_mode = apply_mode_raw.strip().lower()
|
||||
@ -777,6 +784,15 @@ if max_abs_observed_ms > max_abs_skew_ms:
|
||||
)
|
||||
if abs(drift_ms) > max_drift_ms:
|
||||
refusal_reasons.append(f"abs(drift_ms) {abs(drift_ms):.1f} > {max_drift_ms:.1f}")
|
||||
if capture_timebase and capture_timebase.get("valid") is not True:
|
||||
refusal_reasons.append(
|
||||
"capture timebase invalid for calibration: "
|
||||
f"{capture_timebase.get('status', 'unknown')} - {capture_timebase.get('reason', '')}"
|
||||
)
|
||||
elif freshness.get("status") == "invalid":
|
||||
refusal_reasons.append(
|
||||
f"freshness invalid for calibration: {freshness.get('reason', '')}"
|
||||
)
|
||||
|
||||
audio_target_offset_us = active_audio_offset_us + audio_delta_us
|
||||
video_target_offset_us = active_video_offset_us + video_delta_us
|
||||
@ -801,6 +817,10 @@ artifact = {
|
||||
"measurement_host_role": "lab-attached USB host",
|
||||
"probe_media_origin": "server-generated",
|
||||
"probe_media_path": "server generated signatures -> UVC/UAC sinks -> lab host capture",
|
||||
"injection_scope": "server-final-output-handoff",
|
||||
"server_pipeline_reference": "generated media PTS before intentional sync delay",
|
||||
"sink_handoff_path": "video CameraRelay::feed; audio Voice::push",
|
||||
"client_uplink_included": False,
|
||||
"report_json": report_path,
|
||||
"correlation_json": correlation_path if pathlib.Path(correlation_path).exists() else "",
|
||||
"audio_after_video_positive": True,
|
||||
@ -1444,6 +1464,27 @@ def analyze_smoothness(path, report, timeline):
|
||||
video_timestamps = ffprobe_frame_timestamps(path)
|
||||
audio_packets = ffprobe_packet_times(path, "a:0")
|
||||
audio_packet_timestamps = [pts for pts, _duration in audio_packets]
|
||||
media_timeline = {
|
||||
"video_first_s": min(video_timestamps) if video_timestamps else None,
|
||||
"video_last_s": max(video_timestamps) if video_timestamps else None,
|
||||
"audio_first_s": min(audio_packet_timestamps) if audio_packet_timestamps else None,
|
||||
"audio_last_s": max(audio_packet_timestamps) if audio_packet_timestamps else None,
|
||||
}
|
||||
media_timeline["video_span_s"] = (
|
||||
media_timeline["video_last_s"] - media_timeline["video_first_s"]
|
||||
if media_timeline["video_first_s"] is not None and media_timeline["video_last_s"] is not None
|
||||
else None
|
||||
)
|
||||
media_timeline["audio_span_s"] = (
|
||||
media_timeline["audio_last_s"] - media_timeline["audio_first_s"]
|
||||
if media_timeline["audio_first_s"] is not None and media_timeline["audio_last_s"] is not None
|
||||
else None
|
||||
)
|
||||
media_timeline["audio_video_span_gap_s"] = (
|
||||
media_timeline["video_span_s"] - media_timeline["audio_span_s"]
|
||||
if media_timeline["video_span_s"] is not None and media_timeline["audio_span_s"] is not None
|
||||
else None
|
||||
)
|
||||
video_ids = continuity_frame_ids(path)
|
||||
if window_start_s is not None and window_end_s is not None and len(video_ids) == len(video_timestamps):
|
||||
video_ids = [
|
||||
@ -1464,6 +1505,7 @@ def analyze_smoothness(path, report, timeline):
|
||||
"scope": "captured RC target media cadence and continuity over the server-generated probe window",
|
||||
"window_start_s": window_start_s,
|
||||
"window_end_s": window_end_s,
|
||||
"media_timeline": media_timeline,
|
||||
"video": {
|
||||
**interval_smoothness(video_timestamps, expected_video_ms, window_start_s, window_end_s),
|
||||
**sequence_smoothness(video_ids),
|
||||
@ -1504,6 +1546,22 @@ def corrected_server_capture_time_s(server, key):
|
||||
return (server_unix_ns + theia_to_tethys_offset_ns - capture_start_unix_ns) / 1_000_000_000.0
|
||||
|
||||
|
||||
def corrected_capture_time_s_from_unix_ns(server_unix_ns):
|
||||
if (
|
||||
server_unix_ns is None
|
||||
or theia_to_tethys_offset_ns is None
|
||||
or capture_start_unix_ns is None
|
||||
):
|
||||
return None
|
||||
return (server_unix_ns + theia_to_tethys_offset_ns - capture_start_unix_ns) / 1_000_000_000.0
|
||||
|
||||
|
||||
def shifted_unix_ns(unix_ns, delta_ms):
|
||||
if unix_ns is None:
|
||||
return None
|
||||
return unix_ns + int(round(delta_ms * 1_000_000.0))
|
||||
|
||||
|
||||
def nearest_server_event_for_observation(observed, used_event_ids):
|
||||
observed_video_s = finite(observed.get("video_time_s"))
|
||||
observed_audio_s = finite(observed.get("audio_time_s"))
|
||||
@ -1573,6 +1631,14 @@ for observed in report.get("paired_events", []):
|
||||
server_audio_push_s /= 1_000_000.0
|
||||
server_video_feed_unix_ns = as_int_or_none(server.get("video_feed_unix_ns"))
|
||||
server_audio_push_unix_ns = as_int_or_none(server.get("audio_push_unix_ns"))
|
||||
server_video_pipeline_unix_ns = shifted_unix_ns(
|
||||
server_video_feed_unix_ns,
|
||||
-video_delay_ms,
|
||||
)
|
||||
server_audio_pipeline_unix_ns = shifted_unix_ns(
|
||||
server_audio_push_unix_ns,
|
||||
-audio_delay_ms,
|
||||
)
|
||||
tethys_video_unix_ns = (
|
||||
capture_start_unix_ns + int(round(tethys_video_time_s * 1_000_000_000.0))
|
||||
if capture_start_unix_ns is not None and tethys_video_time_s is not None
|
||||
@ -1593,22 +1659,38 @@ for observed in report.get("paired_events", []):
|
||||
if server_audio_push_unix_ns is not None and theia_to_tethys_offset_ns is not None
|
||||
else None
|
||||
)
|
||||
video_freshness_ms = (
|
||||
corrected_server_video_pipeline_unix_ns = (
|
||||
server_video_pipeline_unix_ns + theia_to_tethys_offset_ns
|
||||
if server_video_pipeline_unix_ns is not None and theia_to_tethys_offset_ns is not None
|
||||
else None
|
||||
)
|
||||
corrected_server_audio_pipeline_unix_ns = (
|
||||
server_audio_pipeline_unix_ns + theia_to_tethys_offset_ns
|
||||
if server_audio_pipeline_unix_ns is not None and theia_to_tethys_offset_ns is not None
|
||||
else None
|
||||
)
|
||||
video_sink_handoff_overhead_ms = (
|
||||
(tethys_video_unix_ns - corrected_server_video_feed_unix_ns) / 1_000_000.0
|
||||
if tethys_video_unix_ns is not None and corrected_server_video_feed_unix_ns is not None
|
||||
else None
|
||||
)
|
||||
audio_freshness_ms = (
|
||||
audio_sink_handoff_overhead_ms = (
|
||||
(tethys_audio_unix_ns - corrected_server_audio_push_unix_ns) / 1_000_000.0
|
||||
if tethys_audio_unix_ns is not None and corrected_server_audio_push_unix_ns is not None
|
||||
else None
|
||||
)
|
||||
video_event_age_ms = (
|
||||
video_freshness_ms + video_delay_ms if video_freshness_ms is not None else None
|
||||
video_freshness_ms = (
|
||||
(tethys_video_unix_ns - corrected_server_video_pipeline_unix_ns) / 1_000_000.0
|
||||
if tethys_video_unix_ns is not None and corrected_server_video_pipeline_unix_ns is not None
|
||||
else None
|
||||
)
|
||||
audio_event_age_ms = (
|
||||
audio_freshness_ms + audio_delay_ms if audio_freshness_ms is not None else None
|
||||
audio_freshness_ms = (
|
||||
(tethys_audio_unix_ns - corrected_server_audio_pipeline_unix_ns) / 1_000_000.0
|
||||
if tethys_audio_unix_ns is not None and corrected_server_audio_pipeline_unix_ns is not None
|
||||
else None
|
||||
)
|
||||
video_event_age_ms = video_freshness_ms
|
||||
audio_event_age_ms = audio_freshness_ms
|
||||
residual_path_skew_ms = (
|
||||
observed_skew_ms - server_feed_delta_ms
|
||||
if observed_skew_ms is not None and server_feed_delta_ms is not None
|
||||
@ -1633,12 +1715,16 @@ for observed in report.get("paired_events", []):
|
||||
"server_audio_push_monotonic_us": server.get("audio_push_monotonic_us"),
|
||||
"server_video_feed_unix_ns": server_video_feed_unix_ns,
|
||||
"server_audio_push_unix_ns": server_audio_push_unix_ns,
|
||||
"server_video_pipeline_unix_ns": server_video_pipeline_unix_ns,
|
||||
"server_audio_pipeline_unix_ns": server_audio_pipeline_unix_ns,
|
||||
"tethys_video_unix_ns": tethys_video_unix_ns,
|
||||
"tethys_audio_unix_ns": tethys_audio_unix_ns,
|
||||
"server_video_feed_s": server_video_feed_s,
|
||||
"server_audio_push_s": server_audio_push_s,
|
||||
"video_freshness_ms": video_freshness_ms,
|
||||
"audio_freshness_ms": audio_freshness_ms,
|
||||
"video_sink_handoff_overhead_ms": video_sink_handoff_overhead_ms,
|
||||
"audio_sink_handoff_overhead_ms": audio_sink_handoff_overhead_ms,
|
||||
"video_event_age_ms": video_event_age_ms,
|
||||
"audio_event_age_ms": audio_event_age_ms,
|
||||
"server_feed_delta_ms": server_feed_delta_ms,
|
||||
@ -1656,12 +1742,18 @@ server_model = fit_linear(joined, "server_feed_delta_ms")
|
||||
residual_model = fit_linear(joined, "residual_path_skew_ms")
|
||||
video_freshness_model = fit_linear(joined, "video_freshness_ms")
|
||||
audio_freshness_model = fit_linear(joined, "audio_freshness_ms")
|
||||
video_sink_handoff_model = fit_linear(joined, "video_sink_handoff_overhead_ms")
|
||||
audio_sink_handoff_model = fit_linear(joined, "audio_sink_handoff_overhead_ms")
|
||||
video_event_age_model = fit_linear(joined, "video_event_age_ms")
|
||||
audio_event_age_model = fit_linear(joined, "audio_event_age_ms")
|
||||
video_freshness_stats = stats(joined, "video_freshness_ms")
|
||||
audio_freshness_stats = stats(joined, "audio_freshness_ms")
|
||||
video_event_age_stats = shift_stats(video_freshness_stats, video_delay_ms)
|
||||
audio_event_age_stats = shift_stats(audio_freshness_stats, audio_delay_ms)
|
||||
video_sink_handoff_stats = stats(joined, "video_sink_handoff_overhead_ms")
|
||||
audio_sink_handoff_stats = stats(joined, "audio_sink_handoff_overhead_ms")
|
||||
video_event_age_stats = dict(video_freshness_stats)
|
||||
audio_event_age_stats = dict(audio_freshness_stats)
|
||||
video_event_age_stats["intentional_delay_ms"] = video_delay_ms
|
||||
audio_event_age_stats["intentional_delay_ms"] = audio_delay_ms
|
||||
server_observed_correlation = correlation(joined, "server_feed_delta_ms", "observed_skew_ms")
|
||||
sync_verdict = report.get("verdict") or {}
|
||||
sync_passed = sync_verdict.get("passed") is True
|
||||
@ -1726,6 +1818,103 @@ event_age_min_values = [
|
||||
if value is not None and math.isfinite(value)
|
||||
]
|
||||
freshness_min_event_age_ms = min(event_age_min_values) if event_age_min_values else None
|
||||
smoothness = analyze_smoothness(capture_path, report, timeline)
|
||||
media_timeline = smoothness.get("media_timeline") or {}
|
||||
|
||||
|
||||
def negative_rows(rows, key, limit_ms):
|
||||
return [
|
||||
row
|
||||
for row in rows
|
||||
if row.get(key) is not None
|
||||
and math.isfinite(row[key])
|
||||
and row[key] < -limit_ms
|
||||
]
|
||||
|
||||
|
||||
video_negative_rows = negative_rows(joined, "video_freshness_ms", clock_uncertainty_ms)
|
||||
audio_negative_rows = negative_rows(joined, "audio_freshness_ms", clock_uncertainty_ms)
|
||||
capture_timebase_status = "valid"
|
||||
capture_timebase_reason = "Tethys observation timestamps are monotonic against server pipeline references"
|
||||
capture_timebase_reasons = []
|
||||
if not joined:
|
||||
capture_timebase_status = "unknown"
|
||||
capture_timebase_reason = "no joined coded events are available"
|
||||
elif not clock_alignment_available:
|
||||
capture_timebase_status = "unknown"
|
||||
capture_timebase_reason = "server/capture clock alignment is unavailable"
|
||||
elif video_negative_rows or audio_negative_rows:
|
||||
capture_timebase_status = "invalid"
|
||||
if video_negative_rows:
|
||||
capture_timebase_reasons.append(
|
||||
f"video observed before server pipeline injection {len(video_negative_rows)} time(s)"
|
||||
)
|
||||
if audio_negative_rows:
|
||||
capture_timebase_reasons.append(
|
||||
f"audio observed before server pipeline injection {len(audio_negative_rows)} time(s)"
|
||||
)
|
||||
|
||||
span_gap_s = finite(media_timeline.get("audio_video_span_gap_s"))
|
||||
if span_gap_s is not None and abs(span_gap_s) > 1.0:
|
||||
relation = "shorter" if span_gap_s > 0 else "longer"
|
||||
capture_timebase_status = "invalid"
|
||||
capture_timebase_reasons.append(
|
||||
f"captured audio stream is {abs(span_gap_s):.3f}s {relation} than video; "
|
||||
"the recording cannot place audio events on the video capture timeline"
|
||||
)
|
||||
if capture_timebase_reasons:
|
||||
capture_timebase_reason = "; ".join(capture_timebase_reasons)
|
||||
|
||||
server_start_capture_time_s = corrected_capture_time_s_from_unix_ns(
|
||||
as_int_or_none(timeline.get("server_start_unix_ns"))
|
||||
)
|
||||
first_timing_row = joined[0] if joined else {}
|
||||
first_event_timing = {
|
||||
"event_id": first_timing_row.get("event_id"),
|
||||
"code": first_timing_row.get("code"),
|
||||
"server_audio_pipeline_capture_s": corrected_capture_time_s_from_unix_ns(
|
||||
first_timing_row.get("server_audio_pipeline_unix_ns")
|
||||
),
|
||||
"server_video_pipeline_capture_s": corrected_capture_time_s_from_unix_ns(
|
||||
first_timing_row.get("server_video_pipeline_unix_ns")
|
||||
),
|
||||
"server_audio_sink_push_capture_s": corrected_capture_time_s_from_unix_ns(
|
||||
first_timing_row.get("server_audio_push_unix_ns")
|
||||
),
|
||||
"server_video_sink_feed_capture_s": corrected_capture_time_s_from_unix_ns(
|
||||
first_timing_row.get("server_video_feed_unix_ns")
|
||||
),
|
||||
"tethys_audio_observed_s": first_timing_row.get("tethys_audio_time_s"),
|
||||
"tethys_video_observed_s": first_timing_row.get("tethys_video_time_s"),
|
||||
"audio_pipeline_freshness_ms": first_timing_row.get("audio_freshness_ms"),
|
||||
"video_pipeline_freshness_ms": first_timing_row.get("video_freshness_ms"),
|
||||
}
|
||||
timing_map = {
|
||||
"schema": "lesavka.output-timing-map.v1",
|
||||
"scope": "capture-relative map of server pipeline reference, sink handoff, and Tethys observation times",
|
||||
"injection_scope": timeline.get("injection_scope", "server-final-output-handoff"),
|
||||
"server_pipeline_reference": timeline.get(
|
||||
"server_pipeline_reference",
|
||||
"generated media PTS before intentional sync delay",
|
||||
),
|
||||
"sink_handoff_path": timeline.get(
|
||||
"sink_handoff_path",
|
||||
"video CameraRelay::feed; audio Voice::push",
|
||||
),
|
||||
"client_uplink_included": bool(timeline.get("client_uplink_included", False)),
|
||||
"freshness_clock_starts_at": "server pipeline reference before intentional sync delay",
|
||||
"capture_start_unix_ns": capture_start_unix_ns,
|
||||
"server_start_unix_ns": as_int_or_none(timeline.get("server_start_unix_ns")),
|
||||
"server_start_capture_time_s": server_start_capture_time_s,
|
||||
"capture_to_server_start_ms": (
|
||||
server_start_capture_time_s * 1000.0 if server_start_capture_time_s is not None else None
|
||||
),
|
||||
"probe_warmup_ms": as_float(timeline.get("warmup_us"), 0.0) / 1000.0,
|
||||
"probe_duration_ms": as_float(timeline.get("duration_us"), 0.0) / 1000.0,
|
||||
"intentional_audio_delay_ms": audio_delay_ms,
|
||||
"intentional_video_delay_ms": video_delay_ms,
|
||||
"first_event": first_event_timing,
|
||||
}
|
||||
if not event_age_p95_values:
|
||||
freshness_status = "unknown"
|
||||
freshness_reason = "clock-aligned server feed and Tethys capture timestamps were not available"
|
||||
@ -1735,18 +1924,21 @@ elif not clock_alignment_available or clock_uncertainty_ms > max_clock_uncertain
|
||||
f"clock uncertainty {clock_uncertainty_ms:.1f} ms exceeds "
|
||||
f"{max_clock_uncertainty_ms:.1f} ms freshness measurement limit"
|
||||
)
|
||||
elif capture_timebase_status == "invalid":
|
||||
freshness_status = "invalid"
|
||||
freshness_reason = f"capture timebase invalid: {capture_timebase_reason}"
|
||||
elif freshness_min_event_age_ms is not None and freshness_min_event_age_ms < -clock_uncertainty_ms:
|
||||
freshness_status = "invalid"
|
||||
freshness_reason = (
|
||||
f"one media stream has impossible negative RC event age beyond clock uncertainty: "
|
||||
f"minimum event age {freshness_min_event_age_ms:.1f} ms, uncertainty "
|
||||
f"one media stream has impossible negative pipeline freshness beyond clock uncertainty: "
|
||||
f"minimum freshness {freshness_min_event_age_ms:.1f} ms, uncertainty "
|
||||
f"{clock_uncertainty_ms:.1f} ms"
|
||||
)
|
||||
elif freshness_worst_event_p95_ms < -clock_uncertainty_ms:
|
||||
freshness_status = "invalid"
|
||||
freshness_reason = (
|
||||
f"freshness was negative beyond clock uncertainty: "
|
||||
f"worst RC event-age p95 {freshness_worst_event_p95_ms:.1f} ms, uncertainty "
|
||||
f"worst pipeline p95 {freshness_worst_event_p95_ms:.1f} ms, uncertainty "
|
||||
f"{clock_uncertainty_ms:.1f} ms"
|
||||
)
|
||||
elif not sync_passed:
|
||||
@ -1760,14 +1952,14 @@ elif freshness_worst_event_with_uncertainty_ms <= max_freshness_age_ms and (
|
||||
):
|
||||
freshness_status = "pass"
|
||||
freshness_reason = (
|
||||
f"worst RC event-age p95 {freshness_worst_event_p95_ms:.1f} ms + clock uncertainty "
|
||||
f"worst pipeline p95 {freshness_worst_event_p95_ms:.1f} ms + clock uncertainty "
|
||||
f"{clock_uncertainty_ms:.1f} ms <= {max_freshness_age_ms:.1f} ms and worst freshness drift "
|
||||
f"{(freshness_worst_drift_ms or 0.0):.1f} ms <= {max_freshness_drift_ms:.1f} ms"
|
||||
)
|
||||
else:
|
||||
freshness_status = "fail"
|
||||
freshness_reason = (
|
||||
f"worst RC event-age p95 "
|
||||
f"worst pipeline p95 "
|
||||
f"{freshness_worst_event_p95_ms if freshness_worst_event_p95_ms is not None else 0.0:.1f} ms "
|
||||
f"+ clock uncertainty {clock_uncertainty_ms:.1f} ms "
|
||||
f"(limit {max_freshness_age_ms:.1f} ms), worst freshness drift "
|
||||
@ -1775,14 +1967,13 @@ else:
|
||||
f"(limit {max_freshness_drift_ms:.1f} ms)"
|
||||
)
|
||||
|
||||
smoothness = analyze_smoothness(capture_path, report, timeline)
|
||||
|
||||
artifact = {
|
||||
"schema": "lesavka.output-delay-correlation.v1",
|
||||
"report_json": report_path,
|
||||
"server_timeline_json": timeline_path,
|
||||
"joined_event_count": len(joined),
|
||||
"audio_after_video_positive": True,
|
||||
"timing_map": timing_map,
|
||||
"observed_skew_model": observed_model,
|
||||
"server_feed_delta_model": server_model,
|
||||
"residual_path_skew_model": residual_model,
|
||||
@ -1790,8 +1981,15 @@ artifact = {
|
||||
"schema": "lesavka.output-freshness-summary.v1",
|
||||
"status": freshness_status,
|
||||
"reason": freshness_reason,
|
||||
"scope": "clock-corrected server feed to Tethys capture event from the same paired signatures",
|
||||
"scope": "clock-corrected server pipeline reference to Tethys observed playback from the same paired signatures",
|
||||
"capture_start_unix_ns": capture_start_unix_ns,
|
||||
"capture_timebase": {
|
||||
"status": capture_timebase_status,
|
||||
"reason": capture_timebase_reason,
|
||||
"valid": capture_timebase_status == "valid",
|
||||
"video_observed_before_injection_count": len(video_negative_rows),
|
||||
"audio_observed_before_injection_count": len(audio_negative_rows),
|
||||
},
|
||||
"clock_alignment": clock_alignment,
|
||||
"clock_uncertainty_ms": clock_uncertainty_ms,
|
||||
"max_age_limit_ms": max_freshness_age_ms,
|
||||
@ -1799,19 +1997,35 @@ artifact = {
|
||||
"max_clock_uncertainty_ms": max_clock_uncertainty_ms,
|
||||
"intentional_audio_delay_ms": audio_delay_ms,
|
||||
"intentional_video_delay_ms": video_delay_ms,
|
||||
"worst_p95_path_freshness_ms": freshness_worst_p95_ms,
|
||||
"worst_p95_sink_handoff_overhead_ms": max(
|
||||
[
|
||||
value
|
||||
for value in [
|
||||
video_sink_handoff_stats.get("p95_ms"),
|
||||
audio_sink_handoff_stats.get("p95_ms"),
|
||||
]
|
||||
if value is not None
|
||||
],
|
||||
default=None,
|
||||
),
|
||||
"worst_p95_pipeline_freshness_ms": freshness_worst_p95_ms,
|
||||
"worst_event_age_p95_ms": freshness_worst_event_p95_ms,
|
||||
"worst_event_age_with_uncertainty_ms": freshness_worst_event_with_uncertainty_ms,
|
||||
"minimum_event_age_ms": freshness_min_event_age_ms,
|
||||
"minimum_pipeline_freshness_ms": freshness_min_event_age_ms,
|
||||
"sync_passed": sync_passed,
|
||||
"worst_p95_freshness_ms": freshness_worst_event_p95_ms,
|
||||
"worst_freshness_drift_ms": freshness_worst_drift_ms,
|
||||
"video_freshness_stats": video_freshness_stats,
|
||||
"audio_freshness_stats": audio_freshness_stats,
|
||||
"video_sink_handoff_overhead_stats": video_sink_handoff_stats,
|
||||
"audio_sink_handoff_overhead_stats": audio_sink_handoff_stats,
|
||||
"video_event_age_stats": video_event_age_stats,
|
||||
"audio_event_age_stats": audio_event_age_stats,
|
||||
"video_freshness_model": video_freshness_model,
|
||||
"audio_freshness_model": audio_freshness_model,
|
||||
"video_sink_handoff_overhead_model": video_sink_handoff_model,
|
||||
"audio_sink_handoff_overhead_model": audio_sink_handoff_model,
|
||||
"video_event_age_model": video_event_age_model,
|
||||
"audio_event_age_model": audio_event_age_model,
|
||||
},
|
||||
@ -1854,10 +2068,14 @@ with pathlib.Path(output_csv_path).open("w", newline="", encoding="utf-8") as ha
|
||||
"server_audio_push_s",
|
||||
"server_video_feed_unix_ns",
|
||||
"server_audio_push_unix_ns",
|
||||
"server_video_pipeline_unix_ns",
|
||||
"server_audio_pipeline_unix_ns",
|
||||
"tethys_video_unix_ns",
|
||||
"tethys_audio_unix_ns",
|
||||
"video_freshness_ms",
|
||||
"audio_freshness_ms",
|
||||
"video_sink_handoff_overhead_ms",
|
||||
"audio_sink_handoff_overhead_ms",
|
||||
"video_event_age_ms",
|
||||
"audio_event_age_ms",
|
||||
"server_feed_delta_ms",
|
||||
@ -1871,6 +2089,19 @@ with pathlib.Path(output_csv_path).open("w", newline="", encoding="utf-8") as ha
|
||||
for row in joined:
|
||||
writer.writerow({key: row.get(key) for key in fieldnames})
|
||||
|
||||
|
||||
def fmt_number(value, unit="", precision=1):
|
||||
if value is None:
|
||||
return "n/a"
|
||||
try:
|
||||
value = float(value)
|
||||
except Exception:
|
||||
return "n/a"
|
||||
if not math.isfinite(value):
|
||||
return "n/a"
|
||||
return f"{value:.{precision}f}{unit}"
|
||||
|
||||
|
||||
lines = [
|
||||
f"Output-delay correlation for {report_path}",
|
||||
f"- joined events: {len(joined)}",
|
||||
@ -1882,19 +2113,32 @@ lines = [
|
||||
f"- server/observed correlation: {server_observed_correlation:+.3f}",
|
||||
f"- server drift share of observed: {server_share:.3f}",
|
||||
"",
|
||||
"Timing map",
|
||||
"- scope: capture-relative server pipeline reference, sink handoff, and Tethys observation",
|
||||
f"- injection point: {timing_map.get('injection_scope')} ({timing_map.get('sink_handoff_path')}); client uplink included={timing_map.get('client_uplink_included')}",
|
||||
f"- freshness clock starts at: {timing_map.get('freshness_clock_starts_at')}",
|
||||
f"- capture to server probe start: {fmt_number(timing_map.get('capture_to_server_start_ms'), ' ms')} (setup/pre-roll, not media latency)",
|
||||
f"- probe warmup before first coded event: {fmt_number(timing_map.get('probe_warmup_ms'), ' ms')}",
|
||||
f"- first event server pipeline refs: audio {fmt_number(first_event_timing.get('server_audio_pipeline_capture_s'), 's', 3)}, video {fmt_number(first_event_timing.get('server_video_pipeline_capture_s'), 's', 3)}",
|
||||
f"- first event sink handoff: audio {fmt_number(first_event_timing.get('server_audio_sink_push_capture_s'), 's', 3)}, video {fmt_number(first_event_timing.get('server_video_sink_feed_capture_s'), 's', 3)}",
|
||||
f"- first event Tethys observation: audio {fmt_number(first_event_timing.get('tethys_audio_observed_s'), 's', 3)}, video {fmt_number(first_event_timing.get('tethys_video_observed_s'), 's', 3)}",
|
||||
f"- first event pipeline freshness: audio {fmt_number(first_event_timing.get('audio_pipeline_freshness_ms'), ' ms')}, video {fmt_number(first_event_timing.get('video_pipeline_freshness_ms'), ' ms')}",
|
||||
"",
|
||||
f"Output freshness for {report_path}",
|
||||
"- scope: clock-corrected server feed to Tethys capture event from the same paired signatures",
|
||||
"- scope: clock-corrected server pipeline reference to Tethys observed playback from the same paired signatures",
|
||||
f"- freshness status: {freshness_status} ({freshness_reason})",
|
||||
f"- capture timebase: {capture_timebase_status} ({capture_timebase_reason})",
|
||||
f"- clock uncertainty: +/-{clock_uncertainty_ms:.1f} ms",
|
||||
f"- intentional sync delays: audio {audio_delay_ms:+.1f} ms, video {video_delay_ms:+.1f} ms",
|
||||
f"- media timestamp path offset: video median {video_freshness_stats.get('median_ms') or 0.0:.1f} ms / p95 {video_freshness_stats.get('p95_ms') or 0.0:.1f} ms / max {video_freshness_stats.get('max_ms') or 0.0:.1f} ms; audio median {audio_freshness_stats.get('median_ms') or 0.0:.1f} ms / p95 {audio_freshness_stats.get('p95_ms') or 0.0:.1f} ms / max {audio_freshness_stats.get('max_ms') or 0.0:.1f} ms",
|
||||
f"- RC target event age: video median {video_event_age_stats.get('median_ms') or 0.0:.1f} ms / p95 {video_event_age_stats.get('p95_ms') or 0.0:.1f} ms / max {video_event_age_stats.get('max_ms') or 0.0:.1f} ms; audio median {audio_event_age_stats.get('median_ms') or 0.0:.1f} ms / p95 {audio_event_age_stats.get('p95_ms') or 0.0:.1f} ms / max {audio_event_age_stats.get('max_ms') or 0.0:.1f} ms",
|
||||
f"- freshness budget: worst RC event-age p95 {(freshness_worst_event_p95_ms or 0.0):.1f} ms + clock uncertainty {clock_uncertainty_ms:.1f} ms = {(freshness_worst_event_with_uncertainty_ms or 0.0):.1f} ms vs limit {max_freshness_age_ms:.1f} ms",
|
||||
f"- video event-age drift: {video_event_age_model.get('drift_ms', 0.0):+.1f} ms over paired events ({video_event_age_model.get('slope_ms_per_s', 0.0):+.3f} ms/s)",
|
||||
f"- audio event-age drift: {audio_event_age_model.get('drift_ms', 0.0):+.1f} ms over paired events ({audio_event_age_model.get('slope_ms_per_s', 0.0):+.3f} ms/s)",
|
||||
f"- sink handoff overhead: video median {fmt_number(video_sink_handoff_stats.get('median_ms'), ' ms')} / p95 {fmt_number(video_sink_handoff_stats.get('p95_ms'), ' ms')} / max {fmt_number(video_sink_handoff_stats.get('max_ms'), ' ms')}; audio median {fmt_number(audio_sink_handoff_stats.get('median_ms'), ' ms')} / p95 {fmt_number(audio_sink_handoff_stats.get('p95_ms'), ' ms')} / max {fmt_number(audio_sink_handoff_stats.get('max_ms'), ' ms')}",
|
||||
f"- pipeline freshness: video median {fmt_number(video_freshness_stats.get('median_ms'), ' ms')} / p95 {fmt_number(video_freshness_stats.get('p95_ms'), ' ms')} / max {fmt_number(video_freshness_stats.get('max_ms'), ' ms')}; audio median {fmt_number(audio_freshness_stats.get('median_ms'), ' ms')} / p95 {fmt_number(audio_freshness_stats.get('p95_ms'), ' ms')} / max {fmt_number(audio_freshness_stats.get('max_ms'), ' ms')}",
|
||||
f"- freshness budget: worst pipeline p95 {fmt_number(freshness_worst_event_p95_ms, ' ms')} + clock uncertainty {clock_uncertainty_ms:.1f} ms = {fmt_number(freshness_worst_event_with_uncertainty_ms, ' ms')} vs limit {max_freshness_age_ms:.1f} ms",
|
||||
f"- video freshness drift: {video_event_age_model.get('drift_ms', 0.0):+.1f} ms over paired events ({video_event_age_model.get('slope_ms_per_s', 0.0):+.3f} ms/s)",
|
||||
f"- audio freshness drift: {audio_event_age_model.get('drift_ms', 0.0):+.1f} ms over paired events ({audio_event_age_model.get('slope_ms_per_s', 0.0):+.3f} ms/s)",
|
||||
"",
|
||||
"Output smoothness",
|
||||
f"- window: {smoothness.get('window_start_s') or 0.0:.3f}s to {smoothness.get('window_end_s') or 0.0:.3f}s",
|
||||
f"- capture media timeline: video span {fmt_number(media_timeline.get('video_span_s'), 's', 3)}, audio span {fmt_number(media_timeline.get('audio_span_s'), 's', 3)}, audio/video span gap {fmt_number(media_timeline.get('audio_video_span_gap_s'), 's', 3)}",
|
||||
f"- video cadence: frames {smoothness.get('video', {}).get('timestamps', 0)}, expected interval {(smoothness.get('video', {}).get('expected_interval_ms') or 0.0):.1f} ms, p95 jitter {(smoothness.get('video', {}).get('jitter_stats', {}).get('p95_jitter_ms') or 0.0):.1f} ms, max interval {(smoothness.get('video', {}).get('interval_stats', {}).get('max_interval_ms') or 0.0):.1f} ms, hiccups {smoothness.get('video', {}).get('hiccup_count', 0)}",
|
||||
f"- video continuity: decoded {smoothness.get('video', {}).get('decoded_frames', 0)}, duplicates {smoothness.get('video', {}).get('duplicate_frames', 0)}, estimated missing {smoothness.get('video', {}).get('estimated_missing_frames', 0)}, undecodable {smoothness.get('video', {}).get('undecodable_frames', 0)}",
|
||||
f"- audio packet cadence: packets {smoothness.get('audio', {}).get('packet_cadence', {}).get('timestamps', 0)}, p95 jitter {(smoothness.get('audio', {}).get('packet_cadence', {}).get('jitter_stats', {}).get('p95_jitter_ms') or 0.0):.1f} ms, max interval {(smoothness.get('audio', {}).get('packet_cadence', {}).get('interval_stats', {}).get('max_interval_ms') or 0.0):.1f} ms, hiccups {smoothness.get('audio', {}).get('packet_cadence', {}).get('hiccup_count', 0)}",
|
||||
@ -2081,6 +2325,7 @@ ssh ${SSH_OPTS} "${TETHYS_HOST}" bash -s -- \
|
||||
"${REMOTE_CAPTURE_STACK}" \
|
||||
"${REMOTE_PULSE_CAPTURE_TOOL}" \
|
||||
"${REMOTE_PULSE_VIDEO_MODE}" \
|
||||
"${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE}" \
|
||||
"${REMOTE_AUDIO_SOURCE}" \
|
||||
"${REMOTE_AUDIO_QUIESCE_USER_AUDIO}" \
|
||||
"${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK}" \
|
||||
@ -2098,11 +2343,12 @@ video_format=$6
|
||||
remote_capture_stack=$7
|
||||
remote_pulse_capture_tool=$8
|
||||
remote_pulse_video_mode=$9
|
||||
remote_audio_source=${10}
|
||||
remote_audio_quiesce_user_audio=${11}
|
||||
remote_capture_allow_alsa_fallback=${12}
|
||||
remote_capture_preroll_discard_seconds=${13}
|
||||
remote_capture_ready_settle_seconds=${14}
|
||||
remote_pulse_audio_anchor_silence=${10}
|
||||
remote_audio_source=${11}
|
||||
remote_audio_quiesce_user_audio=${12}
|
||||
remote_capture_allow_alsa_fallback=${13}
|
||||
remote_capture_preroll_discard_seconds=${14}
|
||||
remote_capture_ready_settle_seconds=${15}
|
||||
|
||||
rm -f "${remote_capture}"
|
||||
|
||||
@ -2234,6 +2480,14 @@ gst_video_decode_chain() {
|
||||
esac
|
||||
}
|
||||
|
||||
gst_audio_mixer_element() {
|
||||
if gst-inspect-1.0 audiomixer 2>/dev/null | grep -q 'ignore-inactive-pads'; then
|
||||
printf 'audiomixer name=amix ignore-inactive-pads=true'
|
||||
else
|
||||
printf 'audiomixer name=amix'
|
||||
fi
|
||||
}
|
||||
|
||||
current_video_profile() {
|
||||
if ! command -v v4l2-ctl >/dev/null 2>&1; then
|
||||
return 1
|
||||
@ -2443,6 +2697,7 @@ if [[ -n "${video_format}" ]]; then
|
||||
fi
|
||||
gst_source_caps="$(gst_video_source_caps)"
|
||||
gst_decode_chain="$(gst_video_decode_chain)"
|
||||
gst_audio_mixer="$(gst_audio_mixer_element)"
|
||||
|
||||
run_ffmpeg_capture() {
|
||||
local rc=0
|
||||
@ -2618,40 +2873,85 @@ elif [[ "${capture_mode}" == "pulse" ]]; then
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
gst)
|
||||
case "${remote_pulse_video_mode}" in
|
||||
copy)
|
||||
if [[ "${video_format}" != "mjpeg" && "${video_format}" != "MJPG" && -n "${video_format}" ]]; then
|
||||
printf 'gst copy mode only supports mjpeg input, got %s\n' "${video_format}" >&2
|
||||
exit 64
|
||||
fi
|
||||
run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \
|
||||
gst-launch-1.0 -q -e \
|
||||
matroskamux name=mux ! filesink location="${remote_capture}" \
|
||||
v4l2src device="${resolved_video_device}" do-timestamp=true ! \
|
||||
${gst_source_caps} ! \
|
||||
queue ! mux. \
|
||||
pulsesrc device="${pulse_source}" do-timestamp=true ! \
|
||||
audio/x-raw,rate=48000,channels=2 ! \
|
||||
audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \
|
||||
queue ! mux.
|
||||
;;
|
||||
cfr)
|
||||
run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \
|
||||
gst-launch-1.0 -q -e \
|
||||
matroskamux name=mux ! filesink location="${remote_capture}" \
|
||||
v4l2src device="${resolved_video_device}" do-timestamp=true ! \
|
||||
${gst_source_caps} ! \
|
||||
${gst_decode_chain} \
|
||||
videoconvert ! videorate ! video/x-raw,framerate="${resolved_video_fps}"/1 ! \
|
||||
x264enc tune=zerolatency speed-preset=ultrafast key-int-max=1 bitrate=5000 ! \
|
||||
h264parse ! \
|
||||
queue ! mux. \
|
||||
pulsesrc device="${pulse_source}" do-timestamp=true ! \
|
||||
audio/x-raw,rate=48000,channels=2 ! \
|
||||
audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \
|
||||
queue ! mux.
|
||||
;;
|
||||
gst)
|
||||
if [[ "${remote_pulse_audio_anchor_silence}" == "1" ]]; then
|
||||
printf 'anchoring Pulse capture audio timeline with generated silence\n' >&2
|
||||
fi
|
||||
case "${remote_pulse_video_mode}" in
|
||||
copy)
|
||||
if [[ "${video_format}" != "mjpeg" && "${video_format}" != "MJPG" && -n "${video_format}" ]]; then
|
||||
printf 'gst copy mode only supports mjpeg input, got %s\n' "${video_format}" >&2
|
||||
exit 64
|
||||
fi
|
||||
if [[ "${remote_pulse_audio_anchor_silence}" == "1" ]]; then
|
||||
run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \
|
||||
gst-launch-1.0 -q -e \
|
||||
matroskamux name=mux ! filesink location="${remote_capture}" \
|
||||
v4l2src device="${resolved_video_device}" do-timestamp=true ! \
|
||||
${gst_source_caps} ! \
|
||||
queue ! mux. \
|
||||
${gst_audio_mixer} ! \
|
||||
audio/x-raw,rate=48000,channels=2 ! \
|
||||
queue ! mux. \
|
||||
audiotestsrc wave=silence is-live=true do-timestamp=true ! \
|
||||
audio/x-raw,rate=48000,channels=2 ! \
|
||||
queue ! amix. \
|
||||
pulsesrc device="${pulse_source}" do-timestamp=true ! \
|
||||
audio/x-raw,rate=48000,channels=2 ! \
|
||||
audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \
|
||||
queue ! amix.
|
||||
else
|
||||
run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \
|
||||
gst-launch-1.0 -q -e \
|
||||
matroskamux name=mux ! filesink location="${remote_capture}" \
|
||||
v4l2src device="${resolved_video_device}" do-timestamp=true ! \
|
||||
${gst_source_caps} ! \
|
||||
queue ! mux. \
|
||||
pulsesrc device="${pulse_source}" do-timestamp=true ! \
|
||||
audio/x-raw,rate=48000,channels=2 ! \
|
||||
audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \
|
||||
queue ! mux.
|
||||
fi
|
||||
;;
|
||||
cfr)
|
||||
if [[ "${remote_pulse_audio_anchor_silence}" == "1" ]]; then
|
||||
run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \
|
||||
gst-launch-1.0 -q -e \
|
||||
matroskamux name=mux ! filesink location="${remote_capture}" \
|
||||
v4l2src device="${resolved_video_device}" do-timestamp=true ! \
|
||||
${gst_source_caps} ! \
|
||||
${gst_decode_chain} \
|
||||
videoconvert ! videorate ! video/x-raw,framerate="${resolved_video_fps}"/1 ! \
|
||||
x264enc tune=zerolatency speed-preset=ultrafast key-int-max=1 bitrate=5000 ! \
|
||||
h264parse ! \
|
||||
queue ! mux. \
|
||||
${gst_audio_mixer} ! \
|
||||
audio/x-raw,rate=48000,channels=2 ! \
|
||||
queue ! mux. \
|
||||
audiotestsrc wave=silence is-live=true do-timestamp=true ! \
|
||||
audio/x-raw,rate=48000,channels=2 ! \
|
||||
queue ! amix. \
|
||||
pulsesrc device="${pulse_source}" do-timestamp=true ! \
|
||||
audio/x-raw,rate=48000,channels=2 ! \
|
||||
audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \
|
||||
queue ! amix.
|
||||
else
|
||||
run_tolerant_capture timeout --kill-after=5 --signal=INT "$((capture_seconds + 3))" \
|
||||
gst-launch-1.0 -q -e \
|
||||
matroskamux name=mux ! filesink location="${remote_capture}" \
|
||||
v4l2src device="${resolved_video_device}" do-timestamp=true ! \
|
||||
${gst_source_caps} ! \
|
||||
${gst_decode_chain} \
|
||||
videoconvert ! videorate ! video/x-raw,framerate="${resolved_video_fps}"/1 ! \
|
||||
x264enc tune=zerolatency speed-preset=ultrafast key-int-max=1 bitrate=5000 ! \
|
||||
h264parse ! \
|
||||
queue ! mux. \
|
||||
pulsesrc device="${pulse_source}" do-timestamp=true ! \
|
||||
audio/x-raw,rate=48000,channels=2 ! \
|
||||
audioconvert ! audioresample ! audio/x-raw,rate=48000,channels=2 ! \
|
||||
queue ! mux.
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
printf 'unsupported REMOTE_PULSE_VIDEO_MODE=%s\n' "${remote_pulse_video_mode}" >&2
|
||||
exit 64
|
||||
|
||||
@ -10,7 +10,7 @@ bench = false
|
||||
|
||||
[package]
|
||||
name = "lesavka_server"
|
||||
version = "0.19.21"
|
||||
version = "0.19.22"
|
||||
edition = "2024"
|
||||
autobins = false
|
||||
|
||||
|
||||
@ -152,6 +152,10 @@ struct OutputDelayProbeTimeline {
|
||||
schema: &'static str,
|
||||
origin: &'static str,
|
||||
media_path: &'static str,
|
||||
injection_scope: &'static str,
|
||||
server_pipeline_reference: &'static str,
|
||||
sink_handoff_path: &'static str,
|
||||
client_uplink_included: bool,
|
||||
camera_width: u32,
|
||||
camera_height: u32,
|
||||
camera_fps: u32,
|
||||
@ -208,6 +212,10 @@ impl OutputDelayProbeTimeline {
|
||||
schema: "lesavka.output-delay-server-timeline.v1",
|
||||
origin: "theia-server-generated",
|
||||
media_path: "server generator -> UVC/UAC sinks",
|
||||
injection_scope: "server-final-output-handoff",
|
||||
server_pipeline_reference: "generated media PTS before intentional sync delay",
|
||||
sink_handoff_path: "video CameraRelay::feed; audio Voice::push",
|
||||
client_uplink_included: false,
|
||||
camera_width: camera.width,
|
||||
camera_height: camera.height,
|
||||
camera_fps: camera.fps,
|
||||
@ -388,8 +396,9 @@ fn parse_event_width_codes(raw: &str) -> Result<Vec<u32>> {
|
||||
/// Inputs: the active camera relay, active UAC voice sink, camera profile, and
|
||||
/// probe request timing.
|
||||
/// Outputs: a small count summary after the last generated packet.
|
||||
/// Why: this probe intentionally bypasses client capture/uplink so the measured
|
||||
/// skew is the static output-path difference between server UVC and UAC.
|
||||
/// Why: this probe intentionally bypasses client capture/uplink but uses the
|
||||
/// same final server output handoff calls as received client media, so the
|
||||
/// measured skew/freshness is the server final-handoff-to-RCT path.
|
||||
#[cfg(not(coverage))]
|
||||
pub async fn run_server_output_delay_probe(
|
||||
relay: Arc<CameraRelay>,
|
||||
@ -797,11 +806,12 @@ fn duration_mul(duration: Duration, count: u64) -> Duration {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
DARK_FRAME_RGB, OutputDelayProbeTimeline, ProbeConfig, duration_us, probe_color_for_code,
|
||||
render_audio_chunk, unix_ns_from_start,
|
||||
DARK_FRAME_RGB, EVENT_COLORS, EVENT_FREQUENCIES_HZ, OutputDelayProbeTimeline, ProbeConfig,
|
||||
duration_us, probe_color_for_code, render_audio_chunk, unix_ns_from_start,
|
||||
};
|
||||
use crate::camera::{CameraCodec, CameraConfig, CameraOutput};
|
||||
use lesavka_common::lesavka::OutputDelayProbeRequest;
|
||||
use std::collections::BTreeSet;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
@ -826,6 +836,24 @@ mod tests {
|
||||
assert!(ProbeConfig::from_request(&request).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_probe_signatures_are_unique_for_all_coded_pairs() {
|
||||
assert_eq!(EVENT_COLORS.len(), 16);
|
||||
assert_eq!(EVENT_FREQUENCIES_HZ.len(), 16);
|
||||
|
||||
let colors = EVENT_COLORS
|
||||
.iter()
|
||||
.map(|color| (color.r, color.g, color.b))
|
||||
.collect::<BTreeSet<_>>();
|
||||
let frequencies = EVENT_FREQUENCIES_HZ
|
||||
.iter()
|
||||
.map(|frequency| (*frequency * 10.0).round() as i64)
|
||||
.collect::<BTreeSet<_>>();
|
||||
|
||||
assert_eq!(colors.len(), EVENT_COLORS.len());
|
||||
assert_eq!(frequencies.len(), EVENT_FREQUENCIES_HZ.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_chunk_contains_tone_only_during_coded_pulse() {
|
||||
let config = ProbeConfig::from_request(&OutputDelayProbeRequest {
|
||||
@ -918,6 +946,15 @@ mod tests {
|
||||
);
|
||||
|
||||
let json = serde_json::to_value(timeline).expect("timeline json");
|
||||
assert_eq!(
|
||||
json["injection_scope"].as_str(),
|
||||
Some("server-final-output-handoff")
|
||||
);
|
||||
assert_eq!(
|
||||
json["sink_handoff_path"].as_str(),
|
||||
Some("video CameraRelay::feed; audio Voice::push")
|
||||
);
|
||||
assert_eq!(json["client_uplink_included"].as_bool(), Some(false));
|
||||
assert_eq!(
|
||||
json["server_start_unix_ns"].as_u64(),
|
||||
Some(start_unix_ns as u64)
|
||||
|
||||
@ -58,7 +58,7 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
|
||||
"LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS=${LESAVKA_OUTPUT_FRESHNESS_MAX_CLOCK_UNCERTAINTY_MS:-250}",
|
||||
"REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS=${REMOTE_CAPTURE_PREROLL_DISCARD_SECONDS:-0}",
|
||||
"REMOTE_CAPTURE_READY_SETTLE_SECONDS=${REMOTE_CAPTURE_READY_SETTLE_SECONDS:-1}",
|
||||
"remote_capture_ready_settle_seconds=${14}",
|
||||
"remote_capture_ready_settle_seconds=${15}",
|
||||
"announce_capture_start",
|
||||
"signal_capture_ready",
|
||||
"run_tolerant_capture",
|
||||
@ -95,16 +95,29 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
|
||||
"schema\": \"lesavka.output-delay-correlation.v1\"",
|
||||
"schema\": \"lesavka.clock-alignment.v1\"",
|
||||
"schema\": \"lesavka.output-freshness-summary.v1\"",
|
||||
"schema\": \"lesavka.output-timing-map.v1\"",
|
||||
"server_timeline_json=",
|
||||
"clock_alignment_json: ${LOCAL_CLOCK_ALIGNMENT_JSON}",
|
||||
"dominant_layer",
|
||||
"Timing map",
|
||||
"capture to server probe start",
|
||||
"setup/pre-roll, not media latency",
|
||||
"injection point",
|
||||
"server-final-output-handoff",
|
||||
"video CameraRelay::feed; audio Voice::push",
|
||||
"client uplink included",
|
||||
"freshness clock starts",
|
||||
"freshness status",
|
||||
"clock-corrected server feed to Tethys capture event",
|
||||
"clock-corrected server pipeline reference to Tethys observed playback",
|
||||
"capture timebase",
|
||||
"capture timebase invalid",
|
||||
"intentional sync delays",
|
||||
"media timestamp path offset",
|
||||
"RC target event age",
|
||||
"sink handoff overhead",
|
||||
"pipeline freshness",
|
||||
"freshness budget",
|
||||
"Output smoothness",
|
||||
"capture media timeline",
|
||||
"media_timeline",
|
||||
"video continuity",
|
||||
"audio pilot continuity",
|
||||
"schema\": \"lesavka.output-smoothness-summary.v1\"",
|
||||
@ -114,16 +127,20 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
|
||||
"audio_event_age_ms",
|
||||
"intentional_video_delay_ms",
|
||||
"video_event_age_stats",
|
||||
"video_sink_handoff_overhead_stats",
|
||||
"minimum_pipeline_freshness_ms",
|
||||
"worst_event_age_with_uncertainty_ms",
|
||||
"minimum_event_age_ms",
|
||||
"sync_passed",
|
||||
"sync did not pass, so freshness from paired signatures is not trustworthy",
|
||||
"impossible negative RC event age",
|
||||
"impossible negative pipeline freshness",
|
||||
"video_delay_function_candidate",
|
||||
"source\": \"direct-uvc-uac-output-probe\"",
|
||||
"scope\": \"server-output-static-baseline\"",
|
||||
"applies_to\": \"server UVC/UAC gadget output path\"",
|
||||
"measurement_host_role\": \"lab-attached USB host\"",
|
||||
"injection_scope\": \"server-final-output-handoff\"",
|
||||
"client_uplink_included\": False",
|
||||
"probe_media_origin\": \"server-generated\"",
|
||||
"probe_media_path\": \"server generated signatures -> UVC/UAC sinks -> lab host capture\"",
|
||||
"audio_after_video_positive",
|
||||
@ -151,6 +168,10 @@ fn upstream_sync_script_tunnels_auto_server_addr_through_ssh() {
|
||||
"\"${LESAVKA_OUTPUT_DELAY_PROBE_VIDEO_DELAY_US}\"",
|
||||
"REMOTE_PULSE_CAPTURE_TOOL=${REMOTE_PULSE_CAPTURE_TOOL:-gst}",
|
||||
"REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK=${REMOTE_CAPTURE_ALLOW_ALSA_FALLBACK:-0}",
|
||||
"REMOTE_PULSE_AUDIO_ANCHOR_SILENCE=${REMOTE_PULSE_AUDIO_ANCHOR_SILENCE:-1}",
|
||||
"anchoring Pulse capture audio timeline with generated silence",
|
||||
"audiotestsrc wave=silence is-live=true do-timestamp=true",
|
||||
"audiomixer name=amix ignore-inactive-pads=true",
|
||||
"output_delay_calibration_json",
|
||||
"direct UVC/UAC output-delay calibration",
|
||||
"calibration-save-default",
|
||||
@ -288,6 +309,9 @@ fn server_rc_mode_matrix_validates_advertised_uvc_profiles() {
|
||||
"calibration_video_target_offset_us",
|
||||
"calibration_audio_target_offset_us",
|
||||
"calibration:",
|
||||
"capture_timebase_status",
|
||||
"capture timebase invalid",
|
||||
"capture timing:",
|
||||
"signature_coverage",
|
||||
"paired coded signatures",
|
||||
"signature_missing_codes",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user