fix: normalize blind sync timing signals
This commit is contained in:
parent
826bada865
commit
d2f312b14d
12
AGENTS.md
12
AGENTS.md
@ -539,3 +539,15 @@ remains the truth judge and root-cause localizer, not the production dependency.
|
|||||||
- [x] Emit `root-cause-summary.json` from mirrored probe runs to classify failing layers instead of eyeballing raw metrics.
|
- [x] Emit `root-cause-summary.json` from mirrored probe runs to classify failing layers instead of eyeballing raw metrics.
|
||||||
- [x] Add unit tests for apply/refuse/target behavior in the blind healer.
|
- [x] Add unit tests for apply/refuse/target behavior in the blind healer.
|
||||||
- [ ] Next run should identify the failing layer if confirmation still fails: client capture/uplink, network/server receive, server planner, server sink handoff, or external USB/browser/probe boundary.
|
- [ ] Next run should identify the failing layer if confirmation still fails: client capture/uplink, network/server receive, server planner, server sink handoff, or external USB/browser/probe boundary.
|
||||||
|
|
||||||
|
## 0.17.28 Blind Timing Normalization Checklist
|
||||||
|
|
||||||
|
Context: the first preferred confirmation pass showed the probe-calibrate-confirm
|
||||||
|
loop can work, but also revealed two blind-healing blockers: sink handoff samples
|
||||||
|
stayed empty, and client timing skew included a false cross-pipeline PTS offset.
|
||||||
|
|
||||||
|
- [x] Pair server sink handoff samples by planned due time, not raw local PTS, so offset-compensated streams still produce handoff evidence.
|
||||||
|
- [x] Normalize client sidecar capture/send windows onto the shared capture clock using queue delivery age instead of raw per-pipeline packet PTS.
|
||||||
|
- [x] Add tests proving sink handoff survives large offset-compensated local PTS gaps.
|
||||||
|
- [x] Add tests proving audio/video timing metadata no longer copies packet PTS domains into blind sidecar fields.
|
||||||
|
- [ ] Next mirrored run should show non-zero `planner_sink_handoff_window_samples` and much smaller client send/capture p95 skew before trusting blind healing.
|
||||||
|
|||||||
6
Cargo.lock
generated
6
Cargo.lock
generated
@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lesavka_client"
|
name = "lesavka_client"
|
||||||
version = "0.17.27"
|
version = "0.17.28"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-stream",
|
"async-stream",
|
||||||
@ -1686,7 +1686,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lesavka_common"
|
name = "lesavka_common"
|
||||||
version = "0.17.27"
|
version = "0.17.28"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"base64",
|
"base64",
|
||||||
@ -1698,7 +1698,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lesavka_server"
|
name = "lesavka_server"
|
||||||
version = "0.17.27"
|
version = "0.17.28"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"base64",
|
"base64",
|
||||||
|
|||||||
@ -4,7 +4,7 @@ path = "src/main.rs"
|
|||||||
|
|
||||||
[package]
|
[package]
|
||||||
name = "lesavka_client"
|
name = "lesavka_client"
|
||||||
version = "0.17.27"
|
version = "0.17.28"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
|||||||
@ -500,6 +500,13 @@ fn duration_ms_u32(duration: Duration) -> u32 {
|
|||||||
duration.as_millis().min(u128::from(u32::MAX)) as u32
|
duration.as_millis().min(u128::from(u32::MAX)) as u32
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(not(coverage))]
|
||||||
|
fn shared_capture_window_from_delivery_age(delivery_age: Duration) -> (u64, u64) {
|
||||||
|
let send_pts_us = crate::live_capture_clock::capture_pts_us();
|
||||||
|
let age_us = delivery_age.as_micros().min(u128::from(u64::MAX)) as u64;
|
||||||
|
(send_pts_us.saturating_sub(age_us), send_pts_us)
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(not(coverage))]
|
#[cfg(not(coverage))]
|
||||||
fn attach_audio_timing_metadata(
|
fn attach_audio_timing_metadata(
|
||||||
packet: &mut AudioPacket,
|
packet: &mut AudioPacket,
|
||||||
@ -508,8 +515,9 @@ fn attach_audio_timing_metadata(
|
|||||||
) {
|
) {
|
||||||
static AUDIO_SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
static AUDIO_SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
||||||
packet.seq = AUDIO_SEQUENCE.fetch_add(1, Ordering::Relaxed).saturating_add(1);
|
packet.seq = AUDIO_SEQUENCE.fetch_add(1, Ordering::Relaxed).saturating_add(1);
|
||||||
packet.client_capture_pts_us = packet.pts;
|
let (capture_pts_us, send_pts_us) = shared_capture_window_from_delivery_age(delivery_age);
|
||||||
packet.client_send_pts_us = crate::live_capture_clock::capture_pts_us();
|
packet.client_capture_pts_us = capture_pts_us;
|
||||||
|
packet.client_send_pts_us = send_pts_us;
|
||||||
packet.client_queue_depth = queue_depth_u32(queue_depth);
|
packet.client_queue_depth = queue_depth_u32(queue_depth);
|
||||||
packet.client_queue_age_ms = duration_ms_u32(delivery_age);
|
packet.client_queue_age_ms = duration_ms_u32(delivery_age);
|
||||||
}
|
}
|
||||||
@ -522,8 +530,9 @@ fn attach_video_timing_metadata(
|
|||||||
) {
|
) {
|
||||||
static VIDEO_SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
static VIDEO_SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
||||||
packet.seq = VIDEO_SEQUENCE.fetch_add(1, Ordering::Relaxed).saturating_add(1);
|
packet.seq = VIDEO_SEQUENCE.fetch_add(1, Ordering::Relaxed).saturating_add(1);
|
||||||
packet.client_capture_pts_us = packet.pts;
|
let (capture_pts_us, send_pts_us) = shared_capture_window_from_delivery_age(delivery_age);
|
||||||
packet.client_send_pts_us = crate::live_capture_clock::capture_pts_us();
|
packet.client_capture_pts_us = capture_pts_us;
|
||||||
|
packet.client_send_pts_us = send_pts_us;
|
||||||
packet.client_queue_depth = queue_depth_u32(queue_depth);
|
packet.client_queue_depth = queue_depth_u32(queue_depth);
|
||||||
packet.client_queue_age_ms = duration_ms_u32(delivery_age);
|
packet.client_queue_age_ms = duration_ms_u32(delivery_age);
|
||||||
}
|
}
|
||||||
@ -616,3 +625,54 @@ fn log_uplink_drop(
|
|||||||
limiter.record(reason, count, queue_depth, age_ms);
|
limiter.record(reason, count, queue_depth, age_ms);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod uplink_timing_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn audio_timing_metadata_uses_shared_clock_window_instead_of_packet_pts_domain() {
|
||||||
|
std::thread::sleep(Duration::from_millis(5));
|
||||||
|
let mut packet = AudioPacket {
|
||||||
|
pts: 9_999_999,
|
||||||
|
..AudioPacket::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
attach_audio_timing_metadata(&mut packet, 3, Duration::from_millis(2));
|
||||||
|
|
||||||
|
assert!(packet.seq > 0);
|
||||||
|
assert_eq!(packet.client_queue_depth, 3);
|
||||||
|
assert_eq!(packet.client_queue_age_ms, 2);
|
||||||
|
assert!(
|
||||||
|
packet.client_send_pts_us >= packet.client_capture_pts_us,
|
||||||
|
"send must be on or after the shared-clock capture estimate"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
packet.client_send_pts_us - packet.client_capture_pts_us <= 2_000,
|
||||||
|
"delivery age, not packet PTS domain, should define the timing window"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn video_timing_metadata_uses_shared_clock_window_instead_of_packet_pts_domain() {
|
||||||
|
std::thread::sleep(Duration::from_millis(5));
|
||||||
|
let mut packet = VideoPacket {
|
||||||
|
pts: 9_999_999,
|
||||||
|
..VideoPacket::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
attach_video_timing_metadata(&mut packet, 4, Duration::from_millis(3));
|
||||||
|
|
||||||
|
assert!(packet.seq > 0);
|
||||||
|
assert_eq!(packet.client_queue_depth, 4);
|
||||||
|
assert_eq!(packet.client_queue_age_ms, 3);
|
||||||
|
assert!(
|
||||||
|
packet.client_send_pts_us >= packet.client_capture_pts_us,
|
||||||
|
"send must be on or after the shared-clock capture estimate"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
packet.client_send_pts_us - packet.client_capture_pts_us <= 3_000,
|
||||||
|
"delivery age, not packet PTS domain, should define the timing window"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lesavka_common"
|
name = "lesavka_common"
|
||||||
version = "0.17.27"
|
version = "0.17.28"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
build = "build.rs"
|
build = "build.rs"
|
||||||
|
|
||||||
|
|||||||
@ -10,7 +10,7 @@ bench = false
|
|||||||
|
|
||||||
[package]
|
[package]
|
||||||
name = "lesavka_server"
|
name = "lesavka_server"
|
||||||
version = "0.17.27"
|
version = "0.17.28"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
autobins = false
|
autobins = false
|
||||||
|
|
||||||
|
|||||||
@ -115,12 +115,7 @@ impl UpstreamMediaRuntime {
|
|||||||
.state
|
.state
|
||||||
.lock()
|
.lock()
|
||||||
.expect("upstream media state mutex poisoned");
|
.expect("upstream media state mutex poisoned");
|
||||||
record_presentation_sample(
|
record_presentation_sample(&mut state, UpstreamMediaKind::Microphone, due_at);
|
||||||
&mut state,
|
|
||||||
UpstreamMediaKind::Microphone,
|
|
||||||
local_pts_us,
|
|
||||||
due_at,
|
|
||||||
);
|
|
||||||
state.last_audio_presented_pts_us = Some(local_pts_us);
|
state.last_audio_presented_pts_us = Some(local_pts_us);
|
||||||
if state.phase != UpstreamSyncPhase::Failed {
|
if state.phase != UpstreamSyncPhase::Failed {
|
||||||
state.phase = UpstreamSyncPhase::Live;
|
state.phase = UpstreamSyncPhase::Live;
|
||||||
@ -154,7 +149,7 @@ impl UpstreamMediaRuntime {
|
|||||||
.state
|
.state
|
||||||
.lock()
|
.lock()
|
||||||
.expect("upstream media state mutex poisoned");
|
.expect("upstream media state mutex poisoned");
|
||||||
record_presentation_sample(&mut state, UpstreamMediaKind::Camera, local_pts_us, due_at);
|
record_presentation_sample(&mut state, UpstreamMediaKind::Camera, due_at);
|
||||||
state.last_video_presented_pts_us = Some(local_pts_us);
|
state.last_video_presented_pts_us = Some(local_pts_us);
|
||||||
if state.phase != UpstreamSyncPhase::Failed {
|
if state.phase != UpstreamSyncPhase::Failed {
|
||||||
state.phase = UpstreamSyncPhase::Live;
|
state.phase = UpstreamSyncPhase::Live;
|
||||||
@ -762,11 +757,9 @@ fn record_client_timing_windows(state: &mut UpstreamClockState) {
|
|||||||
fn record_presentation_sample(
|
fn record_presentation_sample(
|
||||||
state: &mut UpstreamClockState,
|
state: &mut UpstreamClockState,
|
||||||
kind: UpstreamMediaKind,
|
kind: UpstreamMediaKind,
|
||||||
local_pts_us: u64,
|
|
||||||
due_at: Instant,
|
due_at: Instant,
|
||||||
) {
|
) {
|
||||||
let sample = state::UpstreamPresentationSample {
|
let sample = state::UpstreamPresentationSample {
|
||||||
local_pts_us,
|
|
||||||
due_at,
|
due_at,
|
||||||
handed_at: Instant::now(),
|
handed_at: Instant::now(),
|
||||||
};
|
};
|
||||||
@ -793,9 +786,8 @@ fn latest_sink_handoff_skew_ms(state: &UpstreamClockState) -> Option<f64> {
|
|||||||
) else {
|
) else {
|
||||||
return None;
|
return None;
|
||||||
};
|
};
|
||||||
let local_pts_delta_ms =
|
let due_at_delta_ms = instant_delta_us(camera.due_at, microphone.due_at).abs() as f64 / 1000.0;
|
||||||
(camera.local_pts_us as i128 - microphone.local_pts_us as i128).abs() as f64 / 1000.0;
|
if due_at_delta_ms > 250.0 {
|
||||||
if local_pts_delta_ms > 250.0 {
|
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
Some(instant_delta_us(camera.handed_at, microphone.handed_at) as f64 / 1000.0)
|
Some(instant_delta_us(camera.handed_at, microphone.handed_at) as f64 / 1000.0)
|
||||||
|
|||||||
@ -13,7 +13,6 @@ pub(super) struct UpstreamTimingSample {
|
|||||||
|
|
||||||
#[derive(Clone, Copy, Debug)]
|
#[derive(Clone, Copy, Debug)]
|
||||||
pub(super) struct UpstreamPresentationSample {
|
pub(super) struct UpstreamPresentationSample {
|
||||||
pub local_pts_us: u64,
|
|
||||||
pub due_at: Instant,
|
pub due_at: Instant,
|
||||||
pub handed_at: Instant,
|
pub handed_at: Instant,
|
||||||
}
|
}
|
||||||
|
|||||||
@ -513,6 +513,29 @@ fn planner_snapshot_tracks_sink_handoff_timing_windows() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial(upstream_media_runtime)]
|
||||||
|
fn sink_handoff_window_pairs_by_due_time_not_offset_local_pts() {
|
||||||
|
let runtime = runtime_without_offsets();
|
||||||
|
let _camera = runtime.activate_camera();
|
||||||
|
let _microphone = runtime.activate_microphone();
|
||||||
|
let due_at = tokio::time::Instant::now()
|
||||||
|
.checked_sub(Duration::from_millis(5))
|
||||||
|
.unwrap_or_else(tokio::time::Instant::now);
|
||||||
|
|
||||||
|
runtime.mark_audio_presented(1_200_000, due_at);
|
||||||
|
std::thread::sleep(Duration::from_millis(1));
|
||||||
|
runtime.mark_video_presented(100_000, due_at);
|
||||||
|
|
||||||
|
let snapshot = runtime.snapshot();
|
||||||
|
|
||||||
|
assert_eq!(snapshot.sink_handoff_window_samples, 1);
|
||||||
|
assert!(
|
||||||
|
snapshot.sink_handoff_skew_ms.is_some(),
|
||||||
|
"offset-compensated streams should still produce handoff evidence when their due times match"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial(upstream_media_runtime)]
|
#[serial(upstream_media_runtime)]
|
||||||
fn planner_snapshot_tracks_client_timing_sidecar_metrics() {
|
fn planner_snapshot_tracks_client_timing_sidecar_metrics() {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user