fix: surface eye capture startup failures
This commit is contained in:
parent
60d10edd03
commit
5c1c24038c
11
AGENTS.md
11
AGENTS.md
@ -644,3 +644,14 @@ verify its own source before asking the analyzer to explain the capture.
|
|||||||
- [x] Retry the Tethys browser recording `/start` request to survive transient SSH banner timeouts.
|
- [x] Retry the Tethys browser recording `/start` request to survive transient SSH banner timeouts.
|
||||||
- [x] Open the manual review capture directory in Dolphin after summarization so copied Tethys captures are immediately inspectable.
|
- [x] Open the manual review capture directory in Dolphin after summarization so copied Tethys captures are immediately inspectable.
|
||||||
- [ ] Re-run the mirrored probe and confirm the preview is visible/audible before trusting any pairing diagnosis.
|
- [ ] Re-run the mirrored probe and confirm the preview is visible/audible before trusting any pairing diagnosis.
|
||||||
|
|
||||||
|
## 0.17.35 Right-Eye Capture Diagnostics Checklist
|
||||||
|
|
||||||
|
Context: manual Tethys testing showed the desktop was awake and HDMI was on, but the right-eye feed stayed black. Server logs showed `eye=r` reached `PLAYING` and then hit a V4L2/GStreamer `Invalid argument (22)` poll error before any frame was pushed, while the left eye streamed normally.
|
||||||
|
|
||||||
|
- [x] Confirm Tethys was not asleep: X11 reported HDMI connected at 1920x1080 and `Monitor is On`.
|
||||||
|
- [x] Remove stale Tethys browser-probe processes after mirrored probe runs so manual Google Meet testing does not compete with old recorder sessions.
|
||||||
|
- [x] Propagate late eye-capture GStreamer bus errors into the gRPC video stream so the launcher reports a preview stream error instead of a silent black window.
|
||||||
|
- [x] Add a first-frame watchdog for eye capture streams so opened-but-empty sources surface as explicit diagnostics.
|
||||||
|
- [ ] Re-run a manual two-eye session and confirm right-eye failures now appear in the session log with the concrete source error.
|
||||||
|
- [ ] If `eye-r` still reports `poll error ... EINVAL`, recover/reseat the right HDMI capture path or add a dedicated eye-capture soft recovery path separate from UVC/UAC.
|
||||||
|
|||||||
6
Cargo.lock
generated
6
Cargo.lock
generated
@ -1652,7 +1652,7 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lesavka_client"
|
name = "lesavka_client"
|
||||||
version = "0.17.34"
|
version = "0.17.35"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-stream",
|
"async-stream",
|
||||||
@ -1686,7 +1686,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lesavka_common"
|
name = "lesavka_common"
|
||||||
version = "0.17.34"
|
version = "0.17.35"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"base64",
|
"base64",
|
||||||
@ -1698,7 +1698,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lesavka_server"
|
name = "lesavka_server"
|
||||||
version = "0.17.34"
|
version = "0.17.35"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"base64",
|
"base64",
|
||||||
|
|||||||
@ -4,7 +4,7 @@ path = "src/main.rs"
|
|||||||
|
|
||||||
[package]
|
[package]
|
||||||
name = "lesavka_client"
|
name = "lesavka_client"
|
||||||
version = "0.17.34"
|
version = "0.17.35"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lesavka_common"
|
name = "lesavka_common"
|
||||||
version = "0.17.34"
|
version = "0.17.35"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
build = "build.rs"
|
build = "build.rs"
|
||||||
|
|
||||||
|
|||||||
@ -10,7 +10,7 @@ bench = false
|
|||||||
|
|
||||||
[package]
|
[package]
|
||||||
name = "lesavka_server"
|
name = "lesavka_server"
|
||||||
version = "0.17.34"
|
version = "0.17.35"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
autobins = false
|
autobins = false
|
||||||
|
|
||||||
|
|||||||
@ -172,9 +172,11 @@ pub async fn eye_ball_with_request(
|
|||||||
let queue_peak_depth = Arc::new(AtomicU32::new(0));
|
let queue_peak_depth = Arc::new(AtomicU32::new(0));
|
||||||
let last_telemetry_sec = Arc::new(AtomicU64::new(0));
|
let last_telemetry_sec = Arc::new(AtomicU64::new(0));
|
||||||
let packet_seq = Arc::new(AtomicU64::new(0));
|
let packet_seq = Arc::new(AtomicU64::new(0));
|
||||||
|
let first_sample_seen = Arc::new(AtomicBool::new(false));
|
||||||
|
|
||||||
let queue_buffers = env_u32("LESAVKA_EYE_QUEUE_BUFFERS", 4).max(1);
|
let queue_buffers = env_u32("LESAVKA_EYE_QUEUE_BUFFERS", 4).max(1);
|
||||||
let appsink_buffers = env_u32("LESAVKA_EYE_APPSINK_BUFFERS", 4).max(1);
|
let appsink_buffers = env_u32("LESAVKA_EYE_APPSINK_BUFFERS", 4).max(1);
|
||||||
|
let first_frame_timeout_ms = env_u32("LESAVKA_EYE_FIRST_FRAME_TIMEOUT_MS", 5_000).max(500);
|
||||||
let keyframe_interval = env_u32("LESAVKA_EYE_KEYFRAME_INTERVAL", request.fps.clamp(1, 5))
|
let keyframe_interval = env_u32("LESAVKA_EYE_KEYFRAME_INTERVAL", request.fps.clamp(1, 5))
|
||||||
.clamp(1, request.fps.max(1));
|
.clamp(1, request.fps.max(1));
|
||||||
let use_test_src =
|
let use_test_src =
|
||||||
@ -225,6 +227,7 @@ pub async fn eye_ball_with_request(
|
|||||||
|
|
||||||
let chan_capacity = env_usize("LESAVKA_EYE_CHAN_CAPACITY", 32).max(8);
|
let chan_capacity = env_usize("LESAVKA_EYE_CHAN_CAPACITY", 32).max(8);
|
||||||
let (tx, rx) = tokio::sync::mpsc::channel(chan_capacity);
|
let (tx, rx) = tokio::sync::mpsc::channel(chan_capacity);
|
||||||
|
let tx_for_bus = tx.clone();
|
||||||
|
|
||||||
if let Some(src_pad) = pipeline
|
if let Some(src_pad) = pipeline
|
||||||
.by_name(&format!("cam_{eye}"))
|
.by_name(&format!("cam_{eye}"))
|
||||||
@ -252,6 +255,7 @@ pub async fn eye_ball_with_request(
|
|||||||
let last_telemetry_sec_for_cb = Arc::clone(&last_telemetry_sec);
|
let last_telemetry_sec_for_cb = Arc::clone(&last_telemetry_sec);
|
||||||
let server_encoder_label_for_cb = server_encoder_label.clone();
|
let server_encoder_label_for_cb = server_encoder_label.clone();
|
||||||
let server_process_cpu_tenths_for_cb = Arc::clone(&server_process_cpu_tenths);
|
let server_process_cpu_tenths_for_cb = Arc::clone(&server_process_cpu_tenths);
|
||||||
|
let first_sample_seen_for_cb = Arc::clone(&first_sample_seen);
|
||||||
sink.set_callbacks(
|
sink.set_callbacks(
|
||||||
gst_app::AppSinkCallbacks::builder()
|
gst_app::AppSinkCallbacks::builder()
|
||||||
.new_sample(move |sink| {
|
.new_sample(move |sink| {
|
||||||
@ -259,6 +263,7 @@ pub async fn eye_ball_with_request(
|
|||||||
let buffer = sample.buffer().ok_or(gst::FlowError::Error)?;
|
let buffer = sample.buffer().ok_or(gst::FlowError::Error)?;
|
||||||
let map = buffer.map_readable().map_err(|_| gst::FlowError::Error)?;
|
let map = buffer.map_readable().map_err(|_| gst::FlowError::Error)?;
|
||||||
let is_idr = contains_idr(map.as_slice());
|
let is_idr = contains_idr(map.as_slice());
|
||||||
|
first_sample_seen_for_cb.store(true, Ordering::Relaxed);
|
||||||
|
|
||||||
static FRAME: AtomicU64 = AtomicU64::new(0);
|
static FRAME: AtomicU64 = AtomicU64::new(0);
|
||||||
let frame = FRAME.fetch_add(1, Ordering::Relaxed);
|
let frame = FRAME.fetch_add(1, Ordering::Relaxed);
|
||||||
@ -406,7 +411,21 @@ pub async fn eye_ball_with_request(
|
|||||||
|
|
||||||
let bus = pipeline.bus().expect("bus");
|
let bus = pipeline.bus().expect("bus");
|
||||||
start_eye_pipeline(&pipeline, &bus, eye)?;
|
start_eye_pipeline(&pipeline, &bus, eye)?;
|
||||||
let bus_watch = BusWatchHandle::spawn(bus, eye.to_owned());
|
let first_sample_seen_for_watchdog = Arc::clone(&first_sample_seen);
|
||||||
|
let tx_for_first_frame_watchdog = tx_for_bus.clone();
|
||||||
|
let first_frame_eye = eye.to_string();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
sleep(Duration::from_millis(first_frame_timeout_ms as u64)).await;
|
||||||
|
if !first_sample_seen_for_watchdog.load(Ordering::Relaxed) {
|
||||||
|
let detail = format!(
|
||||||
|
"eye-{first_frame_eye} capture produced no frames within {first_frame_timeout_ms} ms"
|
||||||
|
);
|
||||||
|
let _ = tx_for_first_frame_watchdog
|
||||||
|
.send(Err(Status::internal(detail)))
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let bus_watch = BusWatchHandle::spawn(bus, eye.to_owned(), tx_for_bus);
|
||||||
|
|
||||||
Ok(VideoStream {
|
Ok(VideoStream {
|
||||||
_pipeline: pipeline,
|
_pipeline: pipeline,
|
||||||
|
|||||||
@ -80,7 +80,11 @@ struct BusWatchHandle {
|
|||||||
|
|
||||||
#[cfg(not(coverage))]
|
#[cfg(not(coverage))]
|
||||||
impl BusWatchHandle {
|
impl BusWatchHandle {
|
||||||
fn spawn(bus: gst::Bus, eye: String) -> Self {
|
fn spawn(
|
||||||
|
bus: gst::Bus,
|
||||||
|
eye: String,
|
||||||
|
stream_errors: tokio::sync::mpsc::Sender<Result<VideoPacket, Status>>,
|
||||||
|
) -> Self {
|
||||||
let alive = Arc::new(AtomicBool::new(true));
|
let alive = Arc::new(AtomicBool::new(true));
|
||||||
let alive_flag = Arc::clone(&alive);
|
let alive_flag = Arc::clone(&alive);
|
||||||
let join = std::thread::spawn(move || {
|
let join = std::thread::spawn(move || {
|
||||||
@ -97,6 +101,12 @@ impl BusWatchHandle {
|
|||||||
err.error(),
|
err.error(),
|
||||||
err.debug().unwrap_or_default()
|
err.debug().unwrap_or_default()
|
||||||
);
|
);
|
||||||
|
let detail = format!(
|
||||||
|
"eye-{eye} capture pipeline error: {} ({})",
|
||||||
|
err.error(),
|
||||||
|
err.debug().unwrap_or_default()
|
||||||
|
);
|
||||||
|
let _ = stream_errors.blocking_send(Err(Status::internal(detail)));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Warning(warning) => {
|
Warning(warning) => {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user