Add configurable overlay obscuring with inpainting

Author: ok-oldking

docs/api_doc/README.md

@@ -49,6 +49,7 @@
     - [任务配置 (Task Configuration)](#任务配置-task-configuration)
     - [屏幕画图 (Screen drawing)](#屏幕画图-screen-drawing)
         - [draw\_boxes](#draw_boxes)
+        - [get\_overlay\_view](#get_overlay_view)
         - [clear\_box](#clear_box)
     - [OCR](#ocr)
         - [ocr](#ocr)
@@ -685,7 +686,7 @@ def get_global_config_desc(self, option) -> str
 
 - **`drop_down`**: 下拉选择框。
     - **参数:** `options` (list[str]): 选项列表。
-    - **可选参数:** `sub_configs` (dict[str, list[str]]): 根据当前下拉选项控制其他配置项是否显示。key 是下拉选项值，value 是需要显示的配置项名称列表；这些配置项会按照列表顺序显示在当前配置项下方，未包含在当前选项列表中的配置项会被隐藏。
+- **`sub_configs`**: 可用于下拉选择框或布尔开关的可选参数。它根据当前值控制其他配置项是否显示；key 是选项值或 `True`/`False`，value 是需要显示的配置项名称列表。这些配置项会按照列表顺序显示在当前配置项下方，未包含在当前值列表中的配置项会被隐藏。
 - **`multi_selection`**: 多选列表。
     - **参数:** `options` (list[str]): 选项列表。
 - **`text_edit`**: 强制使用多行文本框。
@@ -768,6 +769,38 @@ def clear_box(self)
 
 清除屏幕上由 `draw_boxes` 绘制的所有框。
 
+<a name="get_overlay_view"></a>
+
+### get\_overlay\_view
+
+```python
+def get_overlay_view(self)
+```
+
+返回覆盖在捕获窗口上的 Qt overlay widget。`BaseTask` 和 `CustomTab` 可直接调用此方法；配置的
+`my_app` 实例也会获得同名方法。无界面运行时返回 `None`。自定义内容绘制完成后可调用
+`overlay_view.request_show()` 令 overlay 在窗口上显示。
+
+应用配置可提供 `blur_area(width, height)` 回调，返回一个 `Box` 或 `list[Box]`，用于遮挡
+游戏 UID 等静态区域：
+
+```python
+from ok import Box
+
+def blur_area(width, height):
+    return Box(width - 240, height - 42, 240, 42)
+
+config = {
+    'blur_area': blur_area,
+}
+```
+
+配置后，基本设置中会出现 `Enable Blur` 开关，启用后可通过子配置 `Blur Algorithm` 选择
+`Inpaint`（默认）或 `Blur`。`Inpaint` 会使用区域周围的像素重建内容，适用于从较简单背景上移除 UID。
+开启时 overlay 以最多每秒一次的频率显示处理后的区域；保存截图时始终应用所选算法，与该开关
+无关。若还配置了 `screenshot_processor`，它会在处理完成后执行。调试面板中的 `Enable Boxes`
+仅启用框绘制；overlay 只在有框、处理区域或自定义绘制内容需要显示时出现。
+
 ### OCR
 
 <a name="ocr"></a>

ok/__init__.py

@@ -23,7 +23,7 @@
 from ok.gui.MainWindow import MainWindow
 from ok.task.TaskExecutor import TaskExecutor
 from ok.util.Analytics import Analytics
-from ok.util.GlobalConfig import GlobalConfig, basic_options
+from ok.util.GlobalConfig import GlobalConfig, register_basic_options
 from ok.util.clazz import init_class_by_name
 from ok.util.config import Config, ConfigOption
 from ok.util.handler import Handler, ExitEvent
@@ -109,10 +109,6 @@ def __init__(self, config, task_executor,
         else:
             self.to_translate = None
 
-        if self.ok_config.get('use_overlay', False):
-            logger.debug('init overlay')
-            from ok.gui.overlay.OverlayWindow import OverlayWindow
-            self.overlay_window = OverlayWindow(og.device_manager.hwnd_window)
         self.po_translation = None
         if not config.get('window_size'):
             logger.info(f'no config.window_size was set use default')
@@ -129,6 +125,8 @@ def __init__(self, config, task_executor,
 
         if my_app := self.config.get('my_app'):
             og.my_app = init_class_by_name(my_app[0], my_app[1], exit_event)
+            if not hasattr(og.my_app, 'get_overlay_view'):
+                og.my_app.get_overlay_view = self.get_overlay_view
 
         if self.config.get('analytics'):
             self.fire_base_analytics = Analytics(self.config, self.exit_event, og.handler, og.device_manager)
@@ -189,15 +187,24 @@ def show_path_ascii_error(self, path):
         self.show_message_window(title, content)
 
     def update_overlay(self, visible, x, y, window_width, window_height, width, height, scaling):
+        overlay_view = self.get_overlay_view()
+        if overlay_view:
+            overlay_view.update_overlay(visible, x, y, window_width, window_height, width, height, scaling)
 
-        self.overlay_window.update_overlay(visible, x, y, window_width, window_height, width, height, scaling)
+    def get_overlay_view(self):
+        """Return the overlay widget exposed to tasks, custom tabs, and my_app."""
+        if self.overlay_window is None:
+            from ok.gui.overlay.OverlayWindow import OverlayWindow
+            self.overlay_window = OverlayWindow(og.device_manager.hwnd_window)
+            communicate.window.connect(self.overlay_window.update_overlay)
+            self.overlay_window.set_boxes_enabled(self.ok_config.get('use_overlay', False))
+        return self.overlay_window
 
     def show_main_window(self):
         self.do_show_main()
 
     def do_show_main(self):
-        if self.overlay_window:
-            communicate.window.connect(self.overlay_window.update_overlay)
+        self.get_overlay_view()
 
         self.main_window = MainWindow(self, self.config, self.ok_config, self.icon, self.title, self.version,
                                       self.debug,
@@ -263,6 +270,8 @@ def __init__(self, config, exit_event=None):
         og.app = self
         if my_app := self.config.get('my_app'):
             og.my_app = init_class_by_name(my_app[0], my_app[1], exit_event)
+            if not hasattr(og.my_app, 'get_overlay_view'):
+                og.my_app.get_overlay_view = self.get_overlay_view
         logger.debug('init headless app end')
 
     def tr(self, key):
@@ -289,6 +298,9 @@ def quit(self):
         if self.exit_event:
             self.exit_event.set()
 
+    def get_overlay_view(self):
+        return None
+
 
 def get_my_id():
     mac = uuid.getnode()
@@ -365,7 +377,7 @@ def __init__(self, config):
                 else:
                     available_methods.append(method)
 
-        self.global_config.get_config(basic_options)
+        register_basic_options(self.global_config, enable_blur=callable(config.get('blur_area')))
         og.global_config = self.global_config
         og.set_use_dml()
         try:
@@ -758,6 +770,11 @@ def set_use_dml(self):
         logger.info(f'use_dml result is {use_dml}')
         self.use_dml = use_dml
 
+    def get_overlay_view(self):
+        if self.app and hasattr(self.app, 'get_overlay_view'):
+            return self.app.get_overlay_view()
+        return None
+
     def get_trial_expire_util_str(self):
         # Convert the timestamp to a datetime object
         expire_date = datetime.fromtimestamp(self.trial_expire)

ok/gui/Communicate.py

@@ -16,6 +16,8 @@ class Communicate(QObject):
     notification = Signal(str, str, bool, bool, str, object)
     executor_paused: Signal = Signal(bool)
     screenshot = Signal(object, str, bool, object)
+    blur_overlay = Signal(object)
+    clear_blur_overlay = Signal()
     adb_devices: Signal = Signal(bool)
     config_validation: Signal = Signal(str)
     tab = Signal(str)

ok/gui/debug/DebugTab.py

@@ -217,12 +217,10 @@ def capture(processor=None):
         try:
             frame = og.device_manager.capture_method.get_frame()
             if frame is not None:
-                if processor:
-                    frame = processor(frame.copy())
                 current_capture = str(og.device_manager.capture_method) + '_' + str(time.time() * 1000)
                 file_path = og.ok.screenshot.generate_screen_shot(frame, og.ok.screenshot.ui_dict,
                                                                   og.ok.screenshot.screenshot_folder,
-                                                                  current_capture, True, None)
+                                                                  current_capture, True, None, processor=processor)
 
                 # Use subprocess.Popen to open the file explorer and select the file
                 subprocess.Popen(r'explorer /select,"{}"'.format(file_path))

ok/gui/debug/OverlayWidget.py

@@ -1,6 +1,6 @@
 import win32api
 from PySide6.QtCore import Qt, QPoint, QTimer, QRectF, QRect
-from PySide6.QtGui import QPainter, QColor, QPen, QFont, QGuiApplication, QBrush
+from PySide6.QtGui import QPainter, QColor, QPen, QFont, QGuiApplication, QBrush, QImage
 from PySide6.QtWidgets import QWidget
 
 from ok import Logger
@@ -38,6 +38,7 @@ def __init__(self):
         screen = QGuiApplication.primaryScreen()
         self.scaling = screen.devicePixelRatio()
         self.logs = []
+        self.blur_images = []
         communicate.log.connect(self.add_log)
 
     def add_log(self, level_no, message):
@@ -159,13 +160,42 @@ def paintEvent(self, event):
         if not self.isVisible():
             return
         painter = QPainter(self)
-        self.paint_border(painter)
-        self.paint_boxes(painter)
-        self.paint_mouse_position(painter)
-        self.paint_logs(painter)
-        self.paint_alt_overlay(painter)
-        if og.config.get('debug_cover_uid'):
-            self.paint_uid_cover(painter)
+        self.paint_blur(painter)
+        boxes_active = getattr(self, '_boxes_enabled', True) and getattr(self, '_boxes_active', False)
+        if boxes_active:
+            self.paint_boxes(painter)
+            self.paint_border(painter)
+            self.paint_mouse_position(painter)
+            self.paint_logs(painter)
+            self.paint_alt_overlay(painter)
+            if og.config.get('debug_cover_uid'):
+                self.paint_uid_cover(painter)
+
+    def set_blur_patches(self, patches):
+        images = []
+        for x, y, width, height, patch in patches:
+            if patch is None or patch.size == 0:
+                continue
+            if len(patch.shape) == 2:
+                image_data = patch.copy()
+                image_format = QImage.Format_Grayscale8
+            else:
+                image_data = patch[:, :, :3][:, :, ::-1].copy()
+                image_format = QImage.Format_RGB888
+            image = QImage(image_data.data, width, height, image_data.strides[0], image_format).copy()
+            images.append((x, y, width, height, image))
+        self.blur_images = images
+        self.update()
+
+    def clear_blur_patches(self):
+        self.blur_images = []
+        self.update()
+
+    def paint_blur(self, painter):
+        frame_ratio = self.frame_ratio()
+        for x, y, width, height, image in self.blur_images:
+            painter.drawImage(QRectF(x * frame_ratio, y * frame_ratio,
+                                     width * frame_ratio, height * frame_ratio), image)
 
     def paint_alt_overlay(self, painter):
         if not getattr(self, '_is_alt_down', False):

ok/gui/debug/Screenshot.py

@@ -12,6 +12,7 @@
 from ok import Box, og
 from ok import Logger
 from ok.gui.Communicate import communicate
+from ok.util.blur import apply_blur_areas, get_blur_algorithm
 from ok.util.file import find_first_existing_file, clear_folder, sanitize_filename, \
     get_relative_path
 
@@ -126,10 +127,10 @@ def _worker(self):
             self.generate_screen_shot(task[0], task[1], task[2], task[3], task[4], task[5])
             self.task_queue.task_done()
 
-    def generate_screen_shot(self, frame, ui_dict, folder, name, show_box, frame_box):
+    def generate_screen_shot(self, frame, ui_dict, folder, name, show_box, frame_box, processor=None):
         if folder is None:
             return
-        pil_image = self.to_pil_image(frame)
+        pil_image = self.to_pil_image(frame, processor=processor)
         if pil_image is None:
             return
 
@@ -180,10 +181,14 @@ def stop(self):
         if self.task_queue is not None:
             self.task_queue.put(None)
 
-    def to_pil_image(self, frame):
+    def to_pil_image(self, frame, processor=None):
         if frame is None:
             return None
-        if processor := og.config.get('screenshot_processor'):
+        frame = apply_blur_areas(frame, og.config.get('blur_area'),
+                                 get_blur_algorithm(getattr(og, 'global_config', None)))
+        if processor is None:
+            processor = og.config.get('screenshot_processor')
+        if processor:
             frame = processor(frame.copy())
         if len(frame.shape) == 2:
             frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)

ok/gui/i18n/en_US.qm

@@ -819,12 +819,12 @@
             <translation>Choose Interaction</translation>
         </message>
         <message>
-            <source>Show Overlay</source>
-            <translation>Show Overlay</translation>
+            <source>Enable Boxes</source>
+            <translation>Enable Boxes</translation>
         </message>
         <message>
-            <source>Hide Overlay</source>
-            <translation>Hide Overlay</translation>
+            <source>Disable Boxes</source>
+            <translation>Disable Boxes</translation>
         </message>
         <message>
             <source>Show Log on Overlay</source>
@@ -1171,6 +1171,30 @@
             <source>Basic Options</source>
             <translation>Basic Options</translation>
         </message>
+        <message>
+            <source>Enable Blur</source>
+            <translation>Enable Blur</translation>
+        </message>
+        <message>
+            <source>Blur Game UID etc to enhance OLED life</source>
+            <translation>Blur Game UID etc to enhance OLED life</translation>
+        </message>
+        <message>
+            <source>Blur Algorithm</source>
+            <translation>Blur Algorithm</translation>
+        </message>
+        <message>
+            <source>Method used to obscure configured areas</source>
+            <translation>Method used to obscure configured areas</translation>
+        </message>
+        <message>
+            <source>Blur</source>
+            <translation>Blur</translation>
+        </message>
+        <message>
+            <source>Inpaint</source>
+            <translation>Inpaint</translation>
+        </message>
         <message>
             <source>Use DirectML</source>
             <translation>Use DirectML</translation>

ok/gui/i18n/en_US.ts

@@ -817,12 +817,12 @@
             <translation>Elegir interacción</translation>
         </message>
         <message>
-            <source>Show Overlay</source>
-            <translation>Mostrar superposición de depuración</translation>
+            <source>Enable Boxes</source>
+            <translation>Activar cuadros</translation>
         </message>
         <message>
-            <source>Hide Overlay</source>
-            <translation>Ocultar superposición de depuración</translation>
+            <source>Disable Boxes</source>
+            <translation>Desactivar cuadros</translation>
         </message>
         <message>
             <source>Show Log on Overlay</source>
@@ -1169,6 +1169,30 @@
             <source>Basic Options</source>
             <translation>Opciones básicas</translation>
         </message>
+        <message>
+            <source>Enable Blur</source>
+            <translation>Activar desenfoque</translation>
+        </message>
+        <message>
+            <source>Blur Game UID etc to enhance OLED life</source>
+            <translation>Desenfoca el UID del juego y otros elementos para prolongar la vida del OLED</translation>
+        </message>
+        <message>
+            <source>Blur Algorithm</source>
+            <translation>Algoritmo de ocultación</translation>
+        </message>
+        <message>
+            <source>Method used to obscure configured areas</source>
+            <translation>Método utilizado para ocultar las áreas configuradas</translation>
+        </message>
+        <message>
+            <source>Blur</source>
+            <translation>Desenfoque</translation>
+        </message>
+        <message>
+            <source>Inpaint</source>
+            <translation>Relleno inteligente</translation>
+        </message>
         <message>
             <source>Use DirectML</source>
             <translation>Usar DirectML</translation>