fix(drawing): basic multi display support (#49)

blindFS · web-flow · commit 9dcd287c95f2 · 2026-06-04T10:53:52.000+08:00
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
@@ -1,3 +1,3 @@
 [toolchain]
 profile = "default"
-channel = "1.93.1"
+channel = "1.94.1"
diff --git a/src/action/word_picker.rs b/src/action/word_picker.rs
@@ -6,7 +6,6 @@ use crate::{
 };
 use objc2::rc::{Retained, autoreleasepool};
 use objc2_app_kit::NSFontAttributeName;
-use objc2_core_foundation::CGSize;
 use objc2_foundation::{NSMutableAttributedString, NSRange};
 use regex::Regex;
 use std::sync::OnceLock;
@@ -45,7 +44,7 @@ pub struct WordPicker {
 }
 
 impl WordPicker {
-    pub fn new(text: String, screen_size: CGSize, theme: GlyphlowTheme, drawer: &UIDrawer) -> Self {
+    pub fn new(text: String, screen_ratio: f64, theme: GlyphlowTheme, drawer: &UIDrawer) -> Self {
         let (word_strings, offsets) = multilingual_split(&text);
         let digits = digits_by_length(word_strings.len());
         let mut words = Vec::new();
@@ -54,7 +53,6 @@ impl WordPicker {
             words.push(Word { text, label });
         }
 
-        let CGSize { width, height } = screen_size;
         let word_picker = Self {
             raw: text,
             words,
@@ -63,7 +61,7 @@ impl WordPicker {
             is_searching: false,
             label_prefix: String::new(),
             text_prefix: String::new(),
-            screen_ratio: width / (height + 0.01),
+            screen_ratio,
             theme,
             matched: Vec::new(),
         };
diff --git a/src/app_engine/drawing.rs b/src/app_engine/drawing.rs
@@ -4,7 +4,6 @@ use crate::{
     TEXT_ACTION_MENU_ITEMS,
     ax_element::{ElementOfInterest, Target},
     config::RoleOfInterest,
-    util::Frame,
 };
 use objc2::rc::autoreleasepool;
 
@@ -28,16 +27,10 @@ impl AppEngine {
     /// Change selected element of interest
     /// Draw/Update the frame box of selected element
     pub(super) fn select(&mut self, eoi: ElementOfInterest) {
-        self.drawer
-            .draw_frame(&eoi.frame.invert_y(self.screen_size.height));
+        self.drawer.draw_frame(&eoi.frame);
         self.selected = Some(eoi);
     }
 
-    pub(super) fn draw_frame_instant(&self, frame: &Frame) {
-        self.drawer
-            .draw_frame_instant(&frame.invert_y(self.screen_size.height));
-    }
-
     /// Draw/Update hint boxes
     pub(super) fn draw_hints(&mut self) {
         autoreleasepool(|_| {
@@ -46,7 +39,7 @@ impl AppEngine {
                     &self.drawer.root,
                     &self.config.theme,
                     self.key_prefix.len(),
-                    self.screen_size,
+                    &self.overlay_frame,
                 );
             }
         })
@@ -69,7 +62,7 @@ impl AppEngine {
                 hb.set_visible(visible);
 
                 if visible {
-                    hb.refresh(prefix_len, self.screen_size, &self.config.theme);
+                    hb.refresh(prefix_len, &self.overlay_frame, &self.config.theme);
                     nothing_visible = false;
                 }
             }
diff --git a/src/app_engine/filtering.rs b/src/app_engine/filtering.rs
@@ -24,7 +24,7 @@ impl AppEngine {
                 hint_boxes_from_frames(
                     len,
                     iter,
-                    &Frame::from_origion(self.screen_size),
+                    &self.overlay_frame,
                     &self.config.theme,
                     self.config.colored_frame_min_size as f64,
                 )
@@ -85,9 +85,7 @@ impl AppEngine {
         self.drawer.clear_menus_instant();
         self.clear_cache();
         // NOTE: for images with parts out of sight
-        let frame = frame
-            .intersect(&Frame::from_origion(self.screen_size))
-            .unwrap_or(frame);
+        let frame = frame.intersect(&self.overlay_frame).unwrap_or(frame);
         match perform_ocr(&frame, &self.config.ocr_languages).await {
             Ok(ocr_res) if !ocr_res.is_empty() => {
                 self.ocr_cache = Some(ocr_res);
diff --git a/src/app_engine/interaction.rs b/src/app_engine/interaction.rs
@@ -4,7 +4,6 @@ use crate::{
     action::{WordPicker, get_dictionary_attributed_string},
     ax_element::{ElementOfInterest, GetAttribute, SetAttribute},
     config::RoleOfInterest,
-    util::Frame,
 };
 use accessibility::{AXUIElement, AXUIElementActions, AXUIElementAttributes};
 use accessibility_sys::{
@@ -32,9 +31,8 @@ impl AppEngine {
         let mouse_loc = NSEvent::mouseLocation();
 
         if self.config.theme.enable_animation {
-            let color = &self.config.theme.hint_bg_color;
             self.drawer
-                .draw_trail(mouse_loc.x, mouse_loc.y, end_x, end_y, color);
+                .draw_trail(mouse_loc.x, mouse_loc.y, end_x, end_y);
         }
         self.simulate_event(&EventType::MouseMove { x: end_x, y: end_y });
     }
@@ -127,8 +125,7 @@ impl AppEngine {
             .and_then(|eoi| eoi.element())
             .and_then(|ele| ele.parent().ok())
         {
-            let screen_frame = Frame::from_origion(self.screen_size);
-            let frame = parent_element.get_frame(screen_frame);
+            let frame = parent_element.get_frame(self.overlay_frame);
             self.select(ElementOfInterest::new(
                 parent_element,
                 None,
@@ -175,12 +172,10 @@ impl AppEngine {
             TextAction::Split => {
                 self.set_mode(Mode::WordPicking);
                 self.clear_cache();
-                let word_picker = WordPicker::new(
-                    text,
-                    self.screen_size,
-                    self.config.theme.clone(),
-                    &self.drawer,
-                );
+                let (w, h) = self.drawer.current_screen_frame.size();
+                let screen_ratio = w / (h + 0.01);
+                let word_picker =
+                    WordPicker::new(text, screen_ratio, self.config.theme.clone(), &self.drawer);
                 self.hint_width = word_picker.digits;
 
                 self.word_picker = Some(word_picker);
diff --git a/src/app_engine/lifecycle.rs b/src/app_engine/lifecycle.rs
@@ -5,7 +5,7 @@ use crate::{
     config::{GlyphlowConfig, RoleOfInterest, VisibilityCheckingLevel},
     os_util::get_focused_window,
     user_interface::{HintBox, hint_label_from_index, resolve_collisions},
-    util::{Frame, digits_by_length},
+    util::digits_by_length,
 };
 use accessibility::AXUIElementAttributes;
 use log::Level;
@@ -20,7 +20,7 @@ const DEBUG_TIMEOUT: u64 = 5;
 
 impl AppEngine {
     pub(super) fn set_mode(&self, mode: Mode) {
-        log::log!(Level::Trace, "Set mode: {mode:?}");
+        log::trace!("Set mode: {mode:?}");
         if let Ok(mut state) = self.state.lock() {
             *state = mode;
         }
@@ -73,16 +73,16 @@ impl AppEngine {
     }
 
     pub(super) fn get_app_window_info(&mut self) {
-        let screen_frame = Frame::from_origion(self.screen_size);
         let Some(app_win_info) = get_focused_window(
-            screen_frame,
+            self.overlay_frame,
             &self.last_app_window_info,
             self.config.electron_initial_wait_ms,
         ) else {
             return;
         };
 
-        self.draw_frame_instant(&app_win_info.frame);
+        self.drawer.select_screen_frame(&app_win_info.frame);
+        self.drawer.draw_frame_instant(&app_win_info.frame);
         self.last_app_window_info = app_win_info;
     }
 
@@ -129,7 +129,7 @@ impl AppEngine {
             let window_frame = if focused_only {
                 self.last_app_window_info.frame
             } else {
-                Frame::from_origion(self.screen_size)
+                self.overlay_frame
             };
             let _ = std::thread::spawn(move || {
                 traverse(safe_root, frame, window_frame, target, vis_level, result_tx);
@@ -142,7 +142,7 @@ impl AppEngine {
     const HINTBOX_FLUSH_BATCH_SIZE: usize = 5;
 
     pub(super) fn activate(&mut self, target: Target) {
-        log::log!(Level::Debug, "Start traversing, target: {target:?}");
+        log::debug!("Start traversing, target: {target:?}");
         self.clear_cache();
         self.drawer.clear_menus();
         let result_rx = self.ui_element_traverse_on_activation(target);
@@ -162,7 +162,7 @@ impl AppEngine {
                 }
             }
         });
-        log::log!(Level::Debug, "Finish traversing");
+        log::debug!("Finish traversing");
     }
 
     fn handle_element_found(
@@ -175,7 +175,7 @@ impl AppEngine {
         if let Some(idx) = self.element_cache.add_by_target(ele, &self.target) {
             let eoi = &self.element_cache.cache[idx];
 
-            let screen_frame = Frame::from_origion(self.screen_size);
+            let screen_frame = self.overlay_frame;
             let frame = eoi.frame.intersect(&screen_frame).unwrap_or(screen_frame);
 
             let (x, y) = frame.center();
@@ -186,7 +186,7 @@ impl AppEngine {
             // Draw frames for large enough elements
             let frame = if w.max(h) >= self.config.colored_frame_min_size as f64 {
                 *color_idx += 1;
-                Some(eoi.frame.invert_y(self.screen_size.height))
+                Some(eoi.frame)
             } else {
                 None
             };
@@ -200,16 +200,14 @@ impl AppEngine {
                 })
                 .flatten();
 
-            let mut hb = HintBox::new(
-                idx,
-                hint_label_from_index(idx, None),
-                x,
-                self.screen_size.height - y,
-                frame,
-                color,
-            );
+            let mut hb = HintBox::new(idx, hint_label_from_index(idx, None), x, y, frame, color);
 
-            hb.draw(&self.drawer.root, &self.config.theme, 0, self.screen_size);
+            hb.draw(
+                &self.drawer.root,
+                &self.config.theme,
+                0,
+                &self.overlay_frame,
+            );
             if need_flush {
                 CATransaction::flush();
             }
diff --git a/src/app_engine/mod.rs b/src/app_engine/mod.rs
@@ -4,12 +4,11 @@ use crate::{
     ax_element::{ElementCache, ElementOfInterest, Target},
     config::{GlyphlowConfig, RoleOfInterest, WorkFlowAction},
     os_util::AppWindowInfo,
-    user_interface::{HintBox, UIDrawer, get_main_screen_size},
+    user_interface::{HintBox, UIDrawer, get_screen_frames},
     util::Frame,
 };
 use log::Level;
 use objc2::MainThreadMarker;
-use objc2_core_foundation::CGSize;
 use std::{
     collections::VecDeque,
     path::PathBuf,
@@ -66,7 +65,7 @@ pub struct AppEngine {
     pub(super) hint_boxes: Vec<HintBox>,
     pub(super) element_cache: ElementCache,
     pub(super) key_prefix: String,
-    pub(super) screen_size: CGSize,
+    pub(super) overlay_frame: Frame,
     pub(super) drawer: UIDrawer,
     /// Which elements of interest to look for
     pub(super) target: Target,
@@ -99,8 +98,9 @@ impl AppEngine {
         timeout_sender: Sender<usize>,
     ) -> Self {
         let mtm = MainThreadMarker::new().expect("Not on main thread");
-        let screen_size = get_main_screen_size(mtm);
-        let drawer = UIDrawer::new(screen_size, mtm, &config.theme);
+        let screen_frames = get_screen_frames(mtm);
+        let overlay_frame = Frame::union_of_frames(&screen_frames);
+        let drawer = UIDrawer::new(screen_frames, overlay_frame, mtm, &config.theme);
 
         Self {
             state,
@@ -114,7 +114,7 @@ impl AppEngine {
             key_prefix: String::new(),
             target: Target::default(),
             hint_width: 0,
-            screen_size,
+            overlay_frame,
             drawer,
             config,
             timeout_sender,
@@ -123,7 +123,7 @@ impl AppEngine {
             temp_file,
             word_picker: None,
             ocr_cache: None,
-            last_app_window_info: AppWindowInfo::default(screen_size),
+            last_app_window_info: AppWindowInfo::default(overlay_frame),
             multi_selection: MultiSeletionState::default(),
             pending_workflow_actions: VecDeque::new(),
         }
@@ -211,7 +211,7 @@ impl AppEngine {
                 if let Some(text) = text_from_clipboard() {
                     self.selected = Some(ElementOfInterest::pseudo(
                         None,
-                        Frame::from_origion(self.screen_size),
+                        self.last_app_window_info.frame,
                     ));
                     self.update_selected_text_and_show_menu(text);
                 } else {
diff --git a/src/os_util.rs b/src/os_util.rs
@@ -9,7 +9,6 @@ use core_foundation::{
 };
 use objc2::rc::Retained;
 use objc2_app_kit::{NSRunningApplication, NSWorkspace};
-use objc2_core_foundation::CGSize;
 
 use crate::{ax_element::GetAttribute, util::Frame};
 
@@ -49,13 +48,13 @@ pub struct AppWindowInfo {
 }
 
 impl AppWindowInfo {
-    pub fn default(screen_size: CGSize) -> Self {
+    pub fn default(screen_frame: Frame) -> Self {
         Self {
             window: AXUIElement::system_wide(),
             bundle_id: String::new(),
             pid: -1,
             is_electron: false,
-            frame: Frame::from_origion(screen_size),
+            frame: screen_frame,
         }
     }
 
@@ -119,7 +118,7 @@ pub fn get_focused_window(
         .bundleIdentifier()
         .map(|s| s.to_string())
         .unwrap_or_default();
-    log::log!(log::Level::Trace, "Focused app bundle id: {:?}", bundle_id);
+    log::trace!("Focused app bundle id: {:?}", bundle_id);
 
     let pid = app.processIdentifier();
     let is_electron = check_is_electron_app(&app).unwrap_or_default();
diff --git a/src/user_interface/animation.rs b/src/user_interface/animation.rs
diff --git a/src/user_interface/drawer.rs b/src/user_interface/drawer.rs
diff --git a/src/user_interface/hint_box.rs b/src/user_interface/hint_box.rs
diff --git a/src/util.rs b/src/util.rs
