From 20cb355aa0ed7f2b6af08e85028c4b68908177d4 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Thu, 16 Apr 2026 15:07:25 -0300 Subject: [PATCH] lesavka: merge live keyboard state across devices --- client/src/input/inputs.rs | 66 ++++- client/src/input/keyboard.rs | 235 ++++++++++-------- testing/tests/client_inputs_contract.rs | 29 ++- testing/tests/client_inputs_extra_contract.rs | 146 ++++++++++- 4 files changed, 365 insertions(+), 111 deletions(-) diff --git a/client/src/input/inputs.rs b/client/src/input/inputs.rs index 8b9753a..773ddac 100644 --- a/client/src/input/inputs.rs +++ b/client/src/input/inputs.rs @@ -16,7 +16,10 @@ use tracing::{debug, info, warn}; use lesavka_common::lesavka::{KeyboardReport, MouseReport}; -use super::{keyboard::KeyboardAggregator, mouse::MouseAggregator}; +use super::{ + keyboard::{KeyboardAggregator, build_keyboard_report, emit_live_keyboard_report}, + mouse::MouseAggregator, +}; use crate::layout::{Layout, apply as apply_layout}; use tokio::sync::mpsc::UnboundedSender; @@ -29,6 +32,7 @@ pub struct InputAggregator { pending_release: bool, pending_kill: bool, pending_keys: HashSet, + last_keyboard_report: [u8; 8], paste_tx: Option>, keyboards: Vec, mice: Vec, @@ -87,6 +91,7 @@ impl InputAggregator { pending_release: false, pending_kill: false, pending_keys: HashSet::new(), + last_keyboard_report: [0; 8], paste_tx, keyboards: Vec::new(), mice: Vec::new(), @@ -280,9 +285,7 @@ impl InputAggregator { #[cfg(coverage)] pub async fn run(&mut self) -> Result<()> { loop { - for kbd in &mut self.keyboards { - kbd.process_events(); - } + self.process_keyboard_updates(); let quick_toggle_now = self.quick_toggle_active(); self.observe_quick_toggle(quick_toggle_now); @@ -332,8 +335,8 @@ impl InputAggregator { self.publish_routing_state_if_changed(); loop { let mut want_kill = false; - for kbd in &mut self.keyboards { - kbd.process_events(); + self.process_keyboard_updates(); + for kbd in &self.keyboards { want_kill |= kbd.magic_kill(); } self.poll_launcher_routing_request(); @@ -442,6 +445,7 @@ impl InputAggregator { } self.released = false; self.pending_release = false; + self.last_keyboard_report = [0; 8]; } fn begin_local_release(&mut self) { @@ -454,6 +458,7 @@ impl InputAggregator { m.set_send(false); } self.pending_release = true; + self.last_keyboard_report = [0; 8]; self.capture_pending_keys(); } @@ -466,6 +471,42 @@ impl InputAggregator { } } + fn process_keyboard_updates(&mut self) { + for index in 0..self.keyboards.len() { + let updates = { + let keyboard = &mut self.keyboards[index]; + keyboard.drain_key_updates() + }; + for update in updates { + if update.swallowed || !self.keyboard_capture_enabled() { + continue; + } + let report = self.build_combined_keyboard_report(); + if report == self.last_keyboard_report { + continue; + } + emit_live_keyboard_report(&self.kbd_tx, update.code, update.value, report); + self.last_keyboard_report = report; + } + } + } + + fn keyboard_capture_enabled(&self) -> bool { + self.keyboards + .iter() + .any(KeyboardAggregator::sending_enabled) + } + + fn build_combined_keyboard_report(&self) -> [u8; 8] { + let mut pressed = HashSet::new(); + for keyboard in &self.keyboards { + for key in keyboard.pressed_keys_snapshot() { + pressed.insert(key); + } + } + build_keyboard_report(pressed.into_iter()) + } + fn quick_toggle_active(&mut self) -> bool { self.quick_toggle_key.is_some_and(|key| { self.keyboards @@ -568,6 +609,9 @@ fn classify_device(dev: &Device) -> DeviceKind { && keyset .is_some_and(|keys| keys.contains(KeyCode::KEY_A) || keys.contains(KeyCode::KEY_ENTER)) { + if should_ignore_keyboard_device(dev) { + return DeviceKind::Other; + } return DeviceKind::Keyboard; } @@ -601,6 +645,9 @@ fn classify_device(dev: &Device) -> DeviceKind { if evbits.contains(EventType::KEY) { if let Some(keys) = dev.supported_keys() { if keys.contains(KeyCode::KEY_A) || keys.contains(KeyCode::KEY_ENTER) { + if should_ignore_keyboard_device(dev) { + return DeviceKind::Other; + } return DeviceKind::Keyboard; } } @@ -642,6 +689,13 @@ enum DeviceKind { Other, } +fn should_ignore_keyboard_device(dev: &Device) -> bool { + let name = dev.name().unwrap_or_default().to_ascii_lowercase(); + name.contains("lesavka") + || name.contains("automation input") + || name.contains("codex-persistent-kbd") +} + /// Resolves the quick-toggle key from env, defaulting to Pause/Break. fn quick_toggle_key_from_env() -> Option { match std::env::var("LESAVKA_INPUT_TOGGLE_KEY") { diff --git a/client/src/input/keyboard.rs b/client/src/input/keyboard.rs index b7e001b..54e06b3 100644 --- a/client/src/input/keyboard.rs +++ b/client/src/input/keyboard.rs @@ -15,6 +15,14 @@ use lesavka_common::lesavka::KeyboardReport; use super::keymap::{is_modifier, keycode_to_usage}; use lesavka_common::hid::append_char_reports; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyboardEventUpdate { + pub code: KeyCode, + pub value: i32, + pub swallowed: bool, + pub report: [u8; 8], +} + pub struct KeyboardAggregator { dev: Device, tx: Sender, @@ -94,104 +102,17 @@ impl KeyboardAggregator { self.send_report([0; 8]); } - #[cfg(coverage)] pub fn process_events(&mut self) { - self.recent_key_presses.clear(); - let Ok(events) = self - .dev - .fetch_events() - .map(|it| it.collect::>()) - else { - return; - }; - - for ev in events { - if ev.event_type() != EventType::KEY { + for update in self.drain_key_updates() { + if update.swallowed { continue; } - let code = KeyCode::new(ev.code()); - let value = ev.value(); - update_pressed_keys(&mut self.pressed_keys, code, value); - if value == 1 { - self.recent_key_presses.insert(code); - } - - let swallowed = self.try_handle_paste_event(code, value); - if !swallowed { - let report = self.build_report(); - self.emit_live_report(code, value, report); - } - } - } - - #[cfg(not(coverage))] - pub fn process_events(&mut self) { - self.recent_key_presses.clear(); - // --- first fetch, then log (avoids aliasing borrow) --- - let events: Vec = match self.dev.fetch_events() { - Ok(it) => it.collect(), - Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => return, - Err(e) => { - if self.dev_mode { - error!("⌨️❌ read error: {e}"); - } - return; - } - }; - - if self.dev_mode && !events.is_empty() { - trace!( - "⌨️ {} kbd evts from {}", - events.len(), - self.dev.name().unwrap_or("?") - ); - } - - for ev in events { - if ev.event_type() != EventType::KEY { - continue; - } - let code = KeyCode::new(ev.code()); - let value = ev.value(); - update_pressed_keys(&mut self.pressed_keys, code, value); - if value == 1 { - self.recent_key_presses.insert(code); - } - - if self.try_handle_paste_event(code, value) { - continue; - } - - let report = self.build_report(); - // Generate a local sequence number for debugging/log-merge only. - let id = SEQ.fetch_add(1, Ordering::Relaxed); - if self.dev_mode { - debug!(seq = id, ?report, "kbd"); - } - self.emit_live_report(code, value, report); + self.emit_live_report(update.code, update.value, update.report); } } fn build_report(&self) -> [u8; 8] { - let mut out = [0u8; 8]; - let mut mods = 0u8; - let mut keys = Vec::new(); - - for &kc in &self.pressed_keys { - if let Some(m) = is_modifier(kc) { - mods |= m; - continue; - } - if let Some(u) = keycode_to_usage(kc) { - keys.push(u); - } - } - - out[0] = mods; - for (i, k) in keys.into_iter().take(6).enumerate() { - out[2 + i] = k - } - out + build_keyboard_report(self.pressed_keys.iter().copied()) } pub fn has_key(&self, kc: KeyCode) -> bool { @@ -206,6 +127,10 @@ impl KeyboardAggregator { self.pressed_keys.iter().copied().collect() } + pub fn sending_enabled(&self) -> bool { + !self.sending_disabled + } + pub fn magic_grab(&self) -> bool { self.has_key(KeyCode::KEY_LEFTCTRL) && self.has_key(KeyCode::KEY_LEFTSHIFT) @@ -243,20 +168,53 @@ impl KeyboardAggregator { if self.sending_disabled { return; } - let _ = self.tx.send(KeyboardReport { - data: report.to_vec(), - }); + send_keyboard_report(&self.tx, report); } fn emit_live_report(&self, code: KeyCode, value: i32, report: [u8; 8]) { - if should_stage_modifier_report(code, value, report) { - self.send_report(modifier_only_report(report[0])); - let delay = live_modifier_delay(); - if !delay.is_zero() { - std::thread::sleep(delay); - } + if self.sending_disabled { + return; } - self.send_report(report); + emit_live_keyboard_report(&self.tx, code, value, report); + } + + pub fn drain_key_updates(&mut self) -> Vec { + self.recent_key_presses.clear(); + let events = self.fetch_events(); + if self.dev_mode && !events.is_empty() { + trace!( + "⌨️ {} kbd evts from {}", + events.len(), + self.dev.name().unwrap_or("?") + ); + } + + let mut updates = Vec::with_capacity(events.len()); + for ev in events { + if ev.event_type() != EventType::KEY { + continue; + } + let code = KeyCode::new(ev.code()); + let value = ev.value(); + update_pressed_keys(&mut self.pressed_keys, code, value); + if value == 1 { + self.recent_key_presses.insert(code); + } + + let swallowed = self.try_handle_paste_event(code, value); + let report = self.build_report(); + let id = SEQ.fetch_add(1, Ordering::Relaxed); + if self.dev_mode { + debug!(seq = id, ?report, code = ?code, value, swallowed, "kbd"); + } + updates.push(KeyboardEventUpdate { + code, + value, + swallowed, + report, + }); + } + updates } #[cfg(coverage)] @@ -478,6 +436,81 @@ impl KeyboardAggregator { } } +#[cfg(coverage)] +impl KeyboardAggregator { + fn fetch_events(&mut self) -> Vec { + self.dev + .fetch_events() + .map(|it| it.collect::>()) + .unwrap_or_default() + } +} + +#[cfg(not(coverage))] +impl KeyboardAggregator { + fn fetch_events(&mut self) -> Vec { + match self.dev.fetch_events() { + Ok(it) => it.collect(), + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => Vec::new(), + Err(e) => { + if self.dev_mode { + error!("⌨️❌ read error: {e}"); + } + Vec::new() + } + } + } +} + +pub fn build_keyboard_report(pressed_keys: I) -> [u8; 8] +where + I: IntoIterator, +{ + let mut out = [0u8; 8]; + let mut mods = 0u8; + let mut keys = Vec::new(); + + for kc in pressed_keys { + if let Some(m) = is_modifier(kc) { + mods |= m; + continue; + } + if let Some(u) = keycode_to_usage(kc) { + keys.push(u); + } + } + + keys.sort_unstable(); + keys.dedup(); + out[0] = mods; + for (i, k) in keys.into_iter().take(6).enumerate() { + out[2 + i] = k; + } + out +} + +pub fn emit_live_keyboard_report( + tx: &Sender, + code: KeyCode, + value: i32, + report: [u8; 8], +) { + if should_stage_modifier_report(code, value, report) { + send_keyboard_report(tx, modifier_only_report(report[0])); + let delay = live_modifier_delay(); + if !delay.is_zero() { + std::thread::sleep(delay); + } + } + send_keyboard_report(tx, report); +} + +pub fn send_keyboard_report(tx: &Sender, report: [u8; 8]) { + let _ = tx.send(KeyboardReport { + data: report.to_vec(), + }); +} + fn paste_rpc_enabled_from_env() -> bool { let rpc_enabled = std::env::var("LESAVKA_PASTE_RPC") .map(|v| v != "0") diff --git a/testing/tests/client_inputs_contract.rs b/testing/tests/client_inputs_contract.rs index 91c1f3f..309c05e 100644 --- a/testing/tests/client_inputs_contract.rs +++ b/testing/tests/client_inputs_contract.rs @@ -50,7 +50,7 @@ mod inputs_contract { let mut vdev = VirtualDevice::builder() .ok()? - .name("lesavka-input-classify-kbd") + .name("input-classify-kbd") .with_keys(&keys) .ok()? .build() @@ -120,6 +120,22 @@ mod inputs_contract { open_virtual_device(&mut vdev) } + fn build_named_keyboard(name: &str) -> Option { + let mut keys = AttributeSet::::new(); + keys.insert(evdev::KeyCode::KEY_A); + keys.insert(evdev::KeyCode::KEY_ENTER); + + let mut vdev = VirtualDevice::builder() + .ok()? + .name(name) + .with_keys(&keys) + .ok()? + .build() + .ok()?; + + open_virtual_device(&mut vdev) + } + fn build_keyboard_pair(name: &str) -> Option<(VirtualDevice, evdev::Device)> { let mut keys = AttributeSet::::new(); keys.insert(evdev::KeyCode::KEY_A); @@ -184,6 +200,17 @@ mod inputs_contract { } } + #[test] + #[serial] + fn classify_device_ignores_synthetic_automation_keyboards() { + if let Some(automation) = build_named_keyboard("Lesavka Automation Input") { + assert!(matches!(classify_device(&automation), DeviceKind::Other)); + } + if let Some(persistent) = build_named_keyboard("codex-persistent-kbd") { + assert!(matches!(classify_device(&persistent), DeviceKind::Other)); + } + } + #[test] fn toggle_grab_switches_into_local_control_mode() { let mut agg = new_aggregator(); diff --git a/testing/tests/client_inputs_extra_contract.rs b/testing/tests/client_inputs_extra_contract.rs index e37853c..0a81368 100644 --- a/testing/tests/client_inputs_extra_contract.rs +++ b/testing/tests/client_inputs_extra_contract.rs @@ -42,10 +42,14 @@ mod inputs_contract_extra { None } - fn build_keyboard_pair(name: &str) -> Option<(VirtualDevice, evdev::Device)> { + fn build_keyboard_pair_with_keys( + name: &str, + keycodes: &[evdev::KeyCode], + ) -> Option<(VirtualDevice, evdev::Device)> { let mut keys = AttributeSet::::new(); - keys.insert(evdev::KeyCode::KEY_A); - keys.insert(evdev::KeyCode::KEY_ENTER); + for keycode in keycodes { + keys.insert(*keycode); + } let mut vdev = VirtualDevice::builder() .ok()? @@ -59,6 +63,10 @@ mod inputs_contract_extra { Some((vdev, dev)) } + fn build_keyboard_pair(name: &str) -> Option<(VirtualDevice, evdev::Device)> { + build_keyboard_pair_with_keys(name, &[evdev::KeyCode::KEY_A, evdev::KeyCode::KEY_ENTER]) + } + #[test] #[serial] fn quick_toggle_detects_tap_when_press_and_release_land_in_same_poll_cycle() { @@ -92,4 +100,136 @@ mod inputs_contract_extra { "tap activation should be consumed after one observation" ); } + + #[test] + #[serial] + fn process_keyboard_updates_merges_modifier_and_key_across_keyboards() { + let Some((mut shift_vdev, shift_dev)) = build_keyboard_pair_with_keys( + "lesavka-input-merge-shift", + &[evdev::KeyCode::KEY_LEFTSHIFT], + ) else { + return; + }; + let Some((mut a_vdev, a_dev)) = + build_keyboard_pair_with_keys("lesavka-input-merge-a", &[evdev::KeyCode::KEY_A]) + else { + return; + }; + + let (kbd_tx, mut rx) = tokio::sync::broadcast::channel(16); + let (mou_tx, _) = tokio::sync::broadcast::channel(16); + let shift_keyboard = KeyboardAggregator::new(shift_dev, false, kbd_tx.clone(), None); + let a_keyboard = KeyboardAggregator::new(a_dev, false, kbd_tx.clone(), None); + + let mut agg = InputAggregator::new(false, kbd_tx, mou_tx, None); + agg.keyboards.push(shift_keyboard); + agg.keyboards.push(a_keyboard); + + shift_vdev + .emit(&[evdev::InputEvent::new( + evdev::EventType::KEY.0, + evdev::KeyCode::KEY_LEFTSHIFT.0, + 1, + )]) + .expect("emit shift"); + a_vdev + .emit(&[evdev::InputEvent::new( + evdev::EventType::KEY.0, + evdev::KeyCode::KEY_A.0, + 1, + )]) + .expect("emit a"); + thread::sleep(std::time::Duration::from_millis(25)); + + temp_env::with_var("LESAVKA_LIVE_MODIFIER_DELAY_MS", Some("0"), || { + agg.process_keyboard_updates(); + }); + + let reports: Vec> = + std::iter::from_fn(|| rx.try_recv().ok().map(|pkt| pkt.data)).collect(); + assert!( + reports.contains(&vec![0x02, 0, 0x04, 0, 0, 0, 0, 0]), + "expected merged shift+A report, got {reports:?}" + ); + } + + #[test] + #[serial] + fn process_keyboard_updates_keeps_overlapping_plain_keys_from_sticking_across_keyboards() { + let Some((mut a_vdev, a_dev)) = + build_keyboard_pair_with_keys("lesavka-input-merge-a-only", &[evdev::KeyCode::KEY_A]) + else { + return; + }; + let Some((mut s_vdev, s_dev)) = + build_keyboard_pair_with_keys("lesavka-input-merge-s-only", &[evdev::KeyCode::KEY_S]) + else { + return; + }; + + let (kbd_tx, mut rx) = tokio::sync::broadcast::channel(32); + let (mou_tx, _) = tokio::sync::broadcast::channel(16); + let a_keyboard = KeyboardAggregator::new(a_dev, false, kbd_tx.clone(), None); + let s_keyboard = KeyboardAggregator::new(s_dev, false, kbd_tx.clone(), None); + + let mut agg = InputAggregator::new(false, kbd_tx, mou_tx, None); + agg.keyboards.push(a_keyboard); + agg.keyboards.push(s_keyboard); + + a_vdev + .emit(&[evdev::InputEvent::new( + evdev::EventType::KEY.0, + evdev::KeyCode::KEY_A.0, + 1, + )]) + .expect("emit a down"); + s_vdev + .emit(&[evdev::InputEvent::new( + evdev::EventType::KEY.0, + evdev::KeyCode::KEY_S.0, + 1, + )]) + .expect("emit s down"); + thread::sleep(std::time::Duration::from_millis(25)); + agg.process_keyboard_updates(); + + a_vdev + .emit(&[evdev::InputEvent::new( + evdev::EventType::KEY.0, + evdev::KeyCode::KEY_A.0, + 0, + )]) + .expect("emit a up"); + s_vdev + .emit(&[evdev::InputEvent::new( + evdev::EventType::KEY.0, + evdev::KeyCode::KEY_S.0, + 0, + )]) + .expect("emit s up"); + thread::sleep(std::time::Duration::from_millis(25)); + agg.process_keyboard_updates(); + + let reports: Vec> = + std::iter::from_fn(|| rx.try_recv().ok().map(|pkt| pkt.data)).collect(); + assert!( + reports.contains(&vec![0, 0, 0x04, 0, 0, 0, 0, 0]), + "expected A-down report, got {reports:?}" + ); + assert!( + reports.iter().any(|pkt| { + let keys = &pkt[2..8]; + keys.contains(&0x04) && keys.contains(&0x16) + }), + "expected merged A+S overlap report, got {reports:?}" + ); + assert!( + reports.contains(&vec![0, 0, 0x16, 0, 0, 0, 0, 0]), + "expected lone S report after A release, got {reports:?}" + ); + assert!( + reports.contains(&vec![0; 8]), + "expected final empty report after both releases, got {reports:?}" + ); + } }