Fix PostMessage hwnd targeting

Author: ok-oldking

ok/device/capture_methods/hwnd_window.py

@@ -84,6 +84,15 @@ def stop(self):
     def _front_hwnd_candidates(self):
         return list(dict.fromkeys(hwnd for hwnd in (self.top_hwnd, self.hwnd) if hwnd))
 
+    def _top_hwnd_info(self, hwnds):
+        if not hwnds:
+            return None
+
+        for hwnd_info in hwnds:
+            if hwnd_info[0] != self.hwnd:
+                return hwnd_info
+        return hwnds[0]
+
     def bring_to_front(self):
         errors = []
         for refreshed in (False, True):
@@ -215,14 +224,15 @@ def do_update_window_size(self):
                 self.real_y_offset = real_y_offset
                 self.real_width = real_width
                 self.real_height = real_height
-                self.top_hwnd = hwnds[0][0] if hwnds else self.hwnd
+                top_hwnd_info = self._top_hwnd_info(hwnds)
+                self.top_hwnd = top_hwnd_info[0] if top_hwnd_info else self.hwnd
                 self.top_offset_x = 0
                 self.top_offset_y = 0
-                if hwnds and len(hwnds) > 0:
+                if top_hwnd_info:
                     bg_hwnd_info = next((w for w in hwnds if w[0] == self.hwnd), None)
                     if bg_hwnd_info:
-                        self.top_offset_x = hwnds[0][4] - bg_hwnd_info[4]
-                        self.top_offset_y = hwnds[0][5] - bg_hwnd_info[5]
+                        self.top_offset_x = top_hwnd_info[4] - bg_hwnd_info[4]
+                        self.top_offset_y = top_hwnd_info[5] - bg_hwnd_info[5]
 
             exists = self.hwnd > 0
             if self.hwnd > 0:

ok/device/interaction_methods/post_message.py

@@ -11,6 +11,7 @@
 
 logger = Logger.get_logger(__name__)
 
+
 class PostMessageInteraction(BaseInteraction):
 
     def __init__(self, capture: BaseCaptureMethod, hwnd_window):
@@ -72,6 +73,7 @@ def input_text(self, text, activate=True):
     def move(self, x, y, down_btn=0):
         long_pos = self.update_mouse_pos(x, y, True)
         self.post(win32con.WM_MOUSEMOVE, down_btn, long_pos)
+        return long_pos
 
     def scroll(self, x, y, scroll_amount):
         self.try_activate()
@@ -137,9 +139,10 @@ def try_activate(self):
     def click(self, x=-1, y=-1, move_back=False, name=None, down_time=0.01, move=True, key="left"):
         super().click(x, y, name=name)
         if move:
-            self.move(x, y)
+            long_position = self.move(x, y)
             time.sleep(down_time)
-        long_position = self.update_mouse_pos(x, y, activate=not move)
+        else:
+            long_position = self.update_mouse_pos(x, y, activate=True)
 
         if key == "left":
             btn_down = win32con.WM_LBUTTONDOWN
@@ -181,75 +184,52 @@ def mouse_down(self, x=-1, y=-1, name=None, key="left"):
 
     def update_mouse_pos(self, x, y, activate=True):
         self.try_activate()
-        
+
         base_hwnd = self.hwnd_window.top_hwnd if self.hwnd_window.top_hwnd else self.hwnd_window.hwnd
-        
+
         if x == -1 or y == -1:
             x, y = getattr(self, 'bg_mouse_pos', (0, 0))
         else:
             x, y = self.hwnd_window.get_top_window_cords(x, y)
             self.bg_mouse_pos = (x, y)
-            
+
         try:
             abs_x, abs_y = win32gui.ClientToScreen(base_hwnd, (int(x), int(y)))
-            
-            # Validate that the click position falls within the boundary of base_hwnd.
-            # If not, search through the available hwnds list for one that contains it.
+
+            target_hwnd = base_hwnd
             hwnds = getattr(self.hwnd_window, 'hwnds', [])
-            if hwnds and len(hwnds) > 1:
+            for hwnd_info in hwnds:
+                candidate = hwnd_info[0]
+                if not win32gui.IsWindow(candidate):
+                    continue
                 try:
-                    base_rect = win32gui.GetWindowRect(base_hwnd)
-                    in_boundary = base_rect[0] <= abs_x < base_rect[2] and base_rect[1] <= abs_y < base_rect[3]
-                    if not in_boundary:
-                        for hwnd_info in hwnds:
-                            candidate = hwnd_info[0]
-                            if candidate == base_hwnd or not win32gui.IsWindow(candidate):
-                                continue
-                            try:
-                                rect = win32gui.GetWindowRect(candidate)
-                                if rect[0] <= abs_x < rect[2] and rect[1] <= abs_y < rect[3]:
-                                    logger.debug(
-                                        f'update_mouse_pos click ({abs_x},{abs_y}) outside base_hwnd {base_hwnd} rect {base_rect}, switching to hwnd {candidate} rect {rect}')
-                                    base_hwnd = candidate
-                                    # Recompute abs coords relative to the new base_hwnd
-                                    local_candidate_x = hwnd_info[4] if len(hwnd_info) > 4 else 0
-                                    local_candidate_y = hwnd_info[5] if len(hwnd_info) > 5 else 0
-                                    # x,y are in original bg-window coords; translate to candidate client coords
-                                    orig_base_info = next((w for w in hwnds if w[0] == (self.hwnd_window.top_hwnd or self.hwnd_window.hwnd)), None)
-                                    if orig_base_info:
-                                        dx = orig_base_info[4] - local_candidate_x
-                                        dy = orig_base_info[5] - local_candidate_y
-                                        abs_x, abs_y = win32gui.ClientToScreen(candidate, (int(x + dx), int(y + dy)))
-                                    break
-                            except Exception:
-                                continue
+                    left = hwnd_info[4]
+                    top = hwnd_info[5]
+                    right = left + hwnd_info[2]
+                    bottom = top + hwnd_info[3]
+                    if left <= abs_x < right and top <= abs_y < bottom:
+                        target_hwnd = candidate
+                        break
                 except Exception:
-                    pass
-            
-            found_child = None
-            if self.hwnd_window.top_hwnd and self.hwnd_window.top_hwnd != self.hwnd_window.hwnd:
-                def find_child_at_point(child, _):
-                    nonlocal found_child
-                    if win32gui.IsWindowVisible(child):
-                        rect = win32gui.GetWindowRect(child)
-                        if rect[0] <= abs_x < rect[2] and rect[1] <= abs_y < rect[3]:
-                            found_child = child
-                            return False
-                    return True
-                    
-                try:
-                    win32gui.EnumChildWindows(base_hwnd, find_child_at_point, None)
-                except Exception: pass
-                logger.debug(f'found_child {found_child}')
-                
-            target_hwnd = found_child if found_child else base_hwnd
+                    continue
             self._dynamic_target_hwnd = target_hwnd
-            
+
             local_x, local_y = win32gui.ScreenToClient(target_hwnd, (abs_x, abs_y))
-            
-            # logger.debug(f'mouse_pos dynamically aimed at {target_hwnd} ({win32gui.GetClassName(target_hwnd)}): {local_x}, {local_y}')
+
+            hwnd_descriptions = []
+            for index, hwnd_info in enumerate(hwnds):
+                candidate = hwnd_info[0]
+                try:
+                    class_name = win32gui.GetClassName(candidate) if win32gui.IsWindow(candidate) else '<invalid>'
+                except Exception as e:
+                    class_name = f'<class error: {e}>'
+                hwnd_descriptions.append(f'{index}:{candidate}({class_name})')
+            logger.debug(
+                f'hwnd_window hwnds hwnd={self.hwnd_window.hwnd} top_hwnd={self.hwnd_window.top_hwnd}: {hwnd_descriptions}')
+            logger.debug(
+                f'mouse_pos dynamically aimed at {target_hwnd} ({win32gui.GetClassName(target_hwnd)}): {local_x}, {local_y}')
             return win32api.MAKELONG(local_x, local_y)
-            
+
         except Exception as e:
             logger.error(f'update_mouse_pos conversion error targeting {base_hwnd}', e)
             self._dynamic_target_hwnd = base_hwnd

ok/task/TaskExecutor.py

@@ -333,17 +333,19 @@ def wait_condition(self, condition, time_out=0, pre_action=None, post_action=Non
             result = condition()
             result_str = list_or_obj_to_str(result)
             if result:
-                logger.debug(
-                    f"found result {result_str} {(time.time() - start):.3f}")
                 if settle_time == -1:
                     settle_time = self.wait_until_settle_time
                 if settle_time > 0:
-                    if settled > 0 and time.time() - settled > settle_time:
+                    now = time.time()
+                    if settled > 0 and now - settled > settle_time:
+                        logger.debug(f"found result {result_str} {(now - start):.3f}")
                         return result
                     if settled == 0:
-                        settled = time.time()
+                        logger.debug(f"found result {result_str} {(now - start):.3f}")
+                        settled = now
                     continue
                 else:
+                    logger.debug(f"found result {result_str} {(time.time() - start):.3f}")
                     return result
             else:
                 settled = 0

ok/util/print_hwnd.py

@@ -1,3 +1,5 @@
+import time
+
 import win32gui
 import win32process
 import psutil
@@ -105,10 +107,9 @@ def traverse_and_print(parent_hwnd=0, depth=0):
 
             # Formatting variables for clean output
             vis_str = "VIS" if info['visible'] else "INV"
-            hwnd_hex = f"{info['hwnd']:08X}"
             size_str = f"{info['width']}x{info['height']}"
 
-            print(f"{indent}[HWND: {hwnd_hex}] | {vis_str} | Size: {size_str:<9} (Area: {info['area']:<7}) | "
+            print(f"{indent}[HWND: {info['hwnd']}] | {vis_str} | Size: {size_str:<9} (Area: {info['area']:<7}) | "
                   f"EXE: {info['exe']:<20} | PID: {info['pid']:<6} | Class: {info['class']} | "
                   f"CMD: {info['cmdline']}")
 

ok/util/window.py

@@ -122,6 +122,75 @@ def is_foreground_window(hwnd):
     return win32gui.IsWindowVisible(hwnd) and win32gui.GetForegroundWindow() == hwnd
 
 
+def sort_hwnds_top_to_bottom(hwnds):
+    if not hwnds:
+        return hwnds
+
+    hwnd_map = {hwnd_info[0]: hwnd_info for hwnd_info in hwnds}
+    ordered = []
+    seen = set()
+
+    try:
+        hwnd = win32gui.GetTopWindow(0)
+        while hwnd:
+            if hwnd in hwnd_map and hwnd not in seen:
+                ordered.append(hwnd_map[hwnd])
+                seen.add(hwnd)
+                if len(seen) == len(hwnd_map):
+                    break
+            hwnd = win32gui.GetWindow(hwnd, win32con.GW_HWNDNEXT)
+    except Exception as e:
+        logger.debug(f'sort_hwnds_top_to_bottom failed: {e}')
+
+    ordered.extend(hwnd_info for hwnd_info in hwnds if hwnd_info[0] not in seen)
+    return ordered
+
+
+def _hwnd_info(hwnd, full_path=None):
+    x, y, _, _, width, height, m_scaling = get_window_bounds(hwnd)
+    text = win32gui.GetWindowText(hwnd)
+    cname = win32gui.GetClassName(hwnd)
+    return hwnd, full_path, width, height, x, y, text, cname, m_scaling
+
+
+def _clickable_child_hwnds(parent_info, known_hwnds, top_hwnd_class=None):
+    parent_hwnd = parent_info[0]
+    full_path = parent_info[1]
+    clickable = []
+
+    def append_children(parent):
+        child = win32gui.GetTopWindow(parent)
+        while child:
+            if child not in known_hwnds and win32gui.IsWindow(child) and win32gui.IsWindowVisible(child) and win32gui.IsWindowEnabled(child):
+                try:
+                    child_info = _hwnd_info(child, full_path)
+                    if child_info[2] > 0 and child_info[3] > 0:
+                        append_children(child)
+                        if top_hwnd_class is None or _match_class_name(child_info[7], top_hwnd_class) >= 0:
+                            clickable.append(child_info)
+                            known_hwnds.add(child)
+                except Exception:
+                    pass
+            child = win32gui.GetWindow(child, win32con.GW_HWNDNEXT)
+
+    try:
+        append_children(parent_hwnd)
+    except Exception as e:
+        logger.debug(f'_clickable_child_hwnds failed for {parent_hwnd}: {e}')
+    return clickable
+
+
+def expand_clickable_hwnds(hwnds, main_hwnd=0, top_hwnd_class=None):
+    expanded = []
+    known_hwnds = {hwnd_info[0] for hwnd_info in hwnds}
+
+    for hwnd_info in hwnds:
+        if len(hwnds) == 1 or hwnd_info[0] != main_hwnd:
+            expanded.extend(_clickable_child_hwnds(hwnd_info, known_hwnds, top_hwnd_class))
+        expanded.append(hwnd_info)
+    return expanded
+
+
 def show_title_bar(hwnd):
     try:
         current_style = win32gui.GetWindowLong(hwnd, win32con.GWL_STYLE)
@@ -351,6 +420,11 @@ def is_match(hwnd, results):
             for top_item in reversed(filtered_top):
                 if top_item[0] != biggest[0] and not any(r[0] == top_item[0] for r in results):
                     results.insert(0, top_item[:9] if len(top_item) > 9 else top_item)
+            sorted_results = sort_hwnds_top_to_bottom(results)
+            if [r[0] for r in sorted_results] != [r[0] for r in results]:
+                logger.debug(f'find_hwnd sorted hwnds by z-order {[r[0] for r in sorted_results]}')
+            results = sorted_results
+            results = expand_clickable_hwnds(results, biggest[0], top_hwnd_class)
 
     x_offset, y_offset, real_width, real_height = 0, 0, biggest[2], biggest[3]
     if class_name is None and frame_aspect_ratio != 0: