From 7f5a565033ce32a2da8c97969b68136afa754473 Mon Sep 17 00:00:00 2001 From: Owen McGirr Date: Tue, 15 Sep 2026 12:52:38 +0100 Subject: [PATCH] Fix point-scan clicks using explicit target coordinates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Carry scan click targets through the input adapter for menu and auto selection. - Prepare macOS move and button events with explicit coordinates, modifiers, click counts and generated-event markers. - Cover stale cursor readback, native event construction and failure cleanup with tests. 🤖 Auto-generated --- src-tauri/src/input.rs | 42 +++++++- src-tauri/src/lib.rs | 2 + src-tauri/src/macos_point_click.rs | 144 +++++++++++++++++++++++++ src-tauri/src/point_scan_activation.rs | 3 +- src-tauri/src/scan_executor.rs | 123 ++++++++++++++++++++- 5 files changed, 306 insertions(+), 8 deletions(-) create mode 100644 src-tauri/src/macos_point_click.rs diff --git a/src-tauri/src/input.rs b/src-tauri/src/input.rs index f11517ce..01315d27 100644 --- a/src-tauri/src/input.rs +++ b/src-tauri/src/input.rs @@ -125,6 +125,16 @@ pub trait InputInjector { fn move_pointer(&mut self, dx: i32, dy: i32) -> Result<(), String>; fn move_pointer_absolute(&mut self, x: i32, y: i32) -> Result<(), String>; fn click_pointer(&mut self, button: MouseButton, click_count: u8) -> Result<(), String>; + fn click_pointer_at( + &mut self, + point: (i32, i32), + button: MouseButton, + count: u8, + _modifiers: &[&str], + ) -> Result<(), String> { + self.move_pointer_absolute(point.0, point.1)?; + self.click_pointer(button, count) + } fn set_pointer_button(&mut self, button: MouseButton, down: bool) -> Result<(), String>; fn scroll(&mut self, dx: i32, dy: i32) -> Result<(), String>; fn set_key(&mut self, key: &str, down: bool) -> Result<(), String>; @@ -364,6 +374,16 @@ impl InputInjector for Enigo { } Ok(()) } + #[cfg(target_os = "macos")] + fn click_pointer_at( + &mut self, + point: (i32, i32), + button: MouseButton, + count: u8, + modifiers: &[&str], + ) -> Result<(), String> { + crate::macos_point_click::post(point, button, count, modifiers) + } fn set_pointer_button(&mut self, button: MouseButton, down: bool) -> Result<(), String> { self.button( pointer_button(button), @@ -596,6 +616,23 @@ impl DesktopInput { } self.injector.move_pointer_absolute(point.0, point.1) } + pub fn click_pointer_at( + &mut self, + point: (i32, i32), + button: MouseButton, + count: u8, + modifiers: &[&str], + ) -> Result<(), String> { + if self.has_active_drag() || self.has_active_switch_session() { + return Err("End switch forwarding or dragging before clicking.".into()); + } + self.held_button = Some(button); + self.injector + .click_pointer_at(point, button, count, modifiers)?; + self.held_button = None; + self.pointer_feedback = Some(PointerFeedback::Click { button, count }); + Ok(()) + } pub fn click_pointer(&mut self, button: MouseButton, click_count: u8) -> Result<(), String> { self.release_held_button()?; self.injector.click_pointer(button, click_count) @@ -1286,10 +1323,7 @@ impl DesktopInput { self.injector.set_key(key, true)?; } if let Some((point, button, count)) = click { - self.move_pointer_absolute(point.0, point.1)?; - self.held_button = Some(button); - self.injector.click_pointer(button, count)?; - self.held_button = None; + self.click_pointer_at(point, button, count, keys)?; } Ok(()) })(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 06ee736a..3c14020f 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -10,6 +10,8 @@ mod macos; #[cfg(target_os = "macos")] mod macos_overlay_window; #[cfg(target_os = "macos")] +mod macos_point_click; +#[cfg(target_os = "macos")] mod macos_relaunch; mod modifier_overlay; mod mouse_repeat; diff --git a/src-tauri/src/macos_point_click.rs b/src-tauri/src/macos_point_click.rs new file mode 100644 index 00000000..d8b9efaa --- /dev/null +++ b/src-tauri/src/macos_point_click.rs @@ -0,0 +1,144 @@ +use core_graphics::{ + event::{CGEvent, CGEventFlags, CGEventTapLocation, CGEventType, CGMouseButton, EventField}, + event_source::{CGEventSource, CGEventSourceStateID}, + geometry::CGPoint, +}; + +use crate::{input::SCAN_EVENT_MARKER, protocol::MouseButton}; + +fn prepare( + point: (i32, i32), + button: MouseButton, + count: u8, + modifiers: &[&str], +) -> Result, String> { + if !(1..=3).contains(&count) { + return Err("Invalid scan click count.".into()); + } + let mut flags = CGEventFlags::empty(); + for modifier in modifiers { + flags |= match *modifier { + "Shift" => CGEventFlags::CGEventFlagShift, + "Ctrl" => CGEventFlags::CGEventFlagControl, + "Alt" => CGEventFlags::CGEventFlagAlternate, + "Meta" => CGEventFlags::CGEventFlagCommand, + _ => return Err("Invalid scan click modifier.".into()), + }; + } + let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) + .map_err(|_| "Scan click event source could not be initialized.")?; + let (button, down, up) = match button { + MouseButton::Left => ( + CGMouseButton::Left, + CGEventType::LeftMouseDown, + CGEventType::LeftMouseUp, + ), + MouseButton::Right => ( + CGMouseButton::Right, + CGEventType::RightMouseDown, + CGEventType::RightMouseUp, + ), + MouseButton::Middle => ( + CGMouseButton::Center, + CGEventType::OtherMouseDown, + CGEventType::OtherMouseUp, + ), + }; + let target = CGPoint::new(point.0 as f64, point.1 as f64); + let create = |kind, click_count| { + let event = CGEvent::new_mouse_event(source.clone(), kind, target, button) + .map_err(|_| "Scan click events could not be prepared.".to_string())?; + event.set_flags(flags); + event.set_integer_value_field(EventField::EVENT_SOURCE_USER_DATA, SCAN_EVENT_MARKER); + event.set_integer_value_field(EventField::MOUSE_EVENT_CLICK_STATE, click_count); + Ok::<_, String>(event) + }; + let mut events = vec![create(CGEventType::MouseMoved, 0)?]; + for click_count in 1..=count { + events.push(create(down, click_count.into())?); + events.push(create(up, click_count.into())?); + } + Ok(events) +} + +pub fn post( + point: (i32, i32), + button: MouseButton, + count: u8, + modifiers: &[&str], +) -> Result<(), String> { + let events = prepare(point, button, count, modifiers)?; + for event in events { + event.post(CGEventTapLocation::HID); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn native_events_keep_explicit_target_order_count_flags_and_marker() { + for point in [(-1800, -400), (3200, 900)] { + for (button, down, up) in [ + ( + MouseButton::Left, + CGEventType::LeftMouseDown, + CGEventType::LeftMouseUp, + ), + ( + MouseButton::Right, + CGEventType::RightMouseDown, + CGEventType::RightMouseUp, + ), + ( + MouseButton::Middle, + CGEventType::OtherMouseDown, + CGEventType::OtherMouseUp, + ), + ] { + for count in 1..=3 { + let events = + prepare(point, button, count, &["Shift", "Ctrl", "Alt", "Meta"]).unwrap(); + assert_eq!(events.len(), 1 + 2 * count as usize); + assert_eq!(events[0].get_type() as u32, CGEventType::MouseMoved as u32); + for (index, event) in events.iter().enumerate() { + assert_eq!( + (event.location().x, event.location().y), + (point.0 as f64, point.1 as f64) + ); + assert_eq!( + event.get_integer_value_field(EventField::EVENT_SOURCE_USER_DATA), + SCAN_EVENT_MARKER + ); + assert_eq!( + event.get_flags(), + CGEventFlags::CGEventFlagShift + | CGEventFlags::CGEventFlagControl + | CGEventFlags::CGEventFlagAlternate + | CGEventFlags::CGEventFlagCommand + ); + if index > 0 { + assert_eq!( + event.get_type() as u32, + if index % 2 == 1 { down } else { up } as u32 + ); + assert_eq!( + event.get_integer_value_field(EventField::MOUSE_EVENT_CLICK_STATE), + index.div_ceil(2) as i64 + ); + } + } + } + } + } + } + + #[test] + fn invalid_requests_fail_before_any_events_can_be_posted() { + assert!(prepare((0, 0), MouseButton::Left, 0, &[]).is_err()); + assert!(prepare((0, 0), MouseButton::Left, 4, &[]).is_err()); + assert!(prepare((0, 0), MouseButton::Left, 1, &["Space"]).is_err()); + } +} diff --git a/src-tauri/src/point_scan_activation.rs b/src-tauri/src/point_scan_activation.rs index a41bd655..05043b25 100644 --- a/src-tauri/src/point_scan_activation.rs +++ b/src-tauri/src/point_scan_activation.rs @@ -11,6 +11,5 @@ pub fn click( if input.has_active_switch_session() || input.has_active_drag() { return Err("End switch forwarding or dragging before using point scan.".into()); } - input.move_pointer_absolute(point.0, point.1)?; - input.click_pointer(crate::protocol::MouseButton::Left, 1) + input.click_pointer_at(point, crate::protocol::MouseButton::Left, 1, &[]) } diff --git a/src-tauri/src/scan_executor.rs b/src-tauri/src/scan_executor.rs index 4040524d..5600f522 100644 --- a/src-tauri/src/scan_executor.rs +++ b/src-tauri/src/scan_executor.rs @@ -27,14 +27,15 @@ pub fn execute( if input.has_active_drag() { return Err("End the active drag before clicking.".into()); } - input.move_pointer_absolute(point.0, point.1)?; - input.click_pointer( + input.click_pointer_at( + point, if right { MouseButton::Right } else { MouseButton::Left }, count, + &[], ) } Request::Scroll { point, dx, dy } => { @@ -208,11 +209,16 @@ pub fn move_to(point: (i32, i32)) -> Result<(), String> { mod tests { use super::*; use crate::scan_menu::Command; + type TargetClick = ((i32, i32), MouseButton, u8, Vec); #[derive(Default)] struct Fake { events: Vec, fail_release: bool, fail_click: bool, + fail_move: bool, + cursor: (i32, i32), + target_clicks: Vec, + legacy_clicks: usize, } impl InputInjector for Fake { fn inject_text(&mut self, _: &str) -> Result<(), String> { @@ -223,9 +229,33 @@ mod tests { } fn move_pointer_absolute(&mut self, x: i32, y: i32) -> Result<(), String> { self.events.push(format!("move {x} {y}")); + if self.fail_move { + return Err("move failed".into()); + } + Ok(()) + } + fn click_pointer_at( + &mut self, + point: (i32, i32), + button: MouseButton, + count: u8, + modifiers: &[&str], + ) -> Result<(), String> { + self.move_pointer_absolute(point.0, point.1)?; + if self.fail_click { + return Err("click failed".into()); + } + self.events.push(format!("click {button:?} {count}")); + self.target_clicks.push(( + point, + button, + count, + modifiers.iter().map(|key| (*key).into()).collect(), + )); Ok(()) } fn click_pointer(&mut self, button: MouseButton, count: u8) -> Result<(), String> { + self.legacy_clicks += 1; self.events.push(format!("click {button:?} {count}")); if self.fail_click { Err("click failed".into()) @@ -265,6 +295,95 @@ mod tests { Ok(()) } } + #[test] + fn every_scan_click_uses_target_even_while_cursor_readback_is_stale() { + for point in [(-1600, -200), (3600, 1200)] { + let mut cases = vec![ + ( + crate::point_workflow::default_click(point), + MouseButton::Left, + 1, + vec![], + ), + ( + Request::Click { + point, + right: true, + count: 1, + }, + MouseButton::Right, + 1, + vec![], + ), + ( + Request::Click { + point, + right: false, + count: 2, + }, + MouseButton::Left, + 2, + vec![], + ), + ]; + for (command, button, count, keys) in [ + (Command::MiddleClick, MouseButton::Middle, 1, vec![]), + (Command::TripleClick, MouseButton::Left, 3, vec![]), + ( + Command::ShiftClick, + MouseButton::Left, + 1, + vec!["Shift".into()], + ), + ( + Command::CtrlClick, + MouseButton::Left, + 1, + vec!["Ctrl".into()], + ), + (Command::AltClick, MouseButton::Left, 1, vec!["Alt".into()]), + ( + Command::MetaClick, + MouseButton::Left, + 1, + vec!["Meta".into()], + ), + ] { + cases.push((Request::Command { command, point }, button, count, keys)); + } + for (request, button, count, keys) in cases { + let mut input = DesktopInput::new(Fake { + cursor: (15, 20), + ..Default::default() + }); + execute(&mut input, request, true).unwrap(); + assert_eq!(input.injector.cursor, (15, 20)); + assert_eq!(input.injector.legacy_clicks, 0); + assert_eq!(input.injector.target_clicks, [(point, button, count, keys)]); + assert!(!input.has_active_drag()); + } + } + } + + #[test] + fn failed_point_move_never_clicks_and_cleanup_releases_owned_input() { + for command in [None, Some(Command::ShiftClick)] { + let mut input = DesktopInput::new(Fake { + fail_move: true, + ..Default::default() + }); + let point = (-900, 400); + let request = command.map_or(crate::point_workflow::default_click(point), |command| { + Request::Command { command, point } + }); + assert!(execute(&mut input, request, true).is_err()); + assert!(input.injector.target_clicks.is_empty()); + assert_eq!(input.injector.legacy_clicks, 0); + input.release_all().unwrap(); + assert!(!input.has_active_drag()); + } + } + #[test] fn scanning_menu_scrolls_all_four_directions_and_repeats_without_leaving_menu() { use crate::point_scan::Config;