v1.21.0: USB port scanning improvements and report fixes

Author: eldinmiller

.cursor/rules/app.mdc

@@ -0,0 +1,21 @@
+---
+description: BR Equipment Control App development rules
+globs: ["**/*.py"]
+alwaysApply: true
+---
+
+# App Development Rules
+
+## Logging and Debug Output
+
+- **NEVER use `print()` for user-facing debug output** - print() goes to stdout which is not visible in the app
+- **ALWAYS use `log_to_terminal()` for debug messages** - this displays in the app's built-in terminal
+- Import with: `from src.logging.terminal import log_to_terminal`
+- Usage: `log_to_terminal("message", shared_gui_refs)` or `log_to_terminal("message", self.gui_refs)`
+- For modules without access to `shared_gui_refs`, return error details to the caller for logging
+
+## Error Handling
+
+- Always surface meaningful error messages to the user via `log_to_terminal()`
+- Include relevant context (file paths, actual error messages) in error output
+- Don't silently swallow exceptions - log them to the app terminal

CHANGELOG.md

@@ -4,6 +4,23 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [1.21.0] - 2025-01-24
+
+### Added
+- **Jinja2 Dependency**: Added explicit `jinja2>=3.0.0` requirement for HTML report template rendering
+
+### Changed
+- **USB Port Scanning**: Background scanner now pre-fetches serial ports every 3 seconds, eliminating "Scanning ports..." delays in connection menus
+- **USB Connection Menu**: Added "Refresh Ports" option and shows "No ports detected" when empty
+- **GUI Queue Polling**: Reduced from 100ms to 50ms for faster response to device messages
+
+### Fixed
+- **Report Filename Sanitization**: Serial numbers containing invalid characters (like "N/A") no longer create subdirectories - invalid filename characters are now replaced with dashes
+- **Report Error Messages**: Now shows actual Python import errors when report modules fail to load, instead of generic "not found" messages
+- **CSV File Sync on Windows**: Data logger now calls `fsync` before closing files to ensure Windows flushes data to disk before report generation
+- **GUI Thread Safety**: Script completion callbacks now properly schedule Tkinter updates on the main thread, preventing potential race conditions
+- **Report Debug Logging**: Enhanced logging for troubleshooting report generation (CSV paths, file locks, handler calls)
+
 ## [1.20.0] - 2025-11-28
 
 ### Added

_version.py

@@ -1 +1 @@
-__version__ = "1.20.0"
+__version__ = "1.21.0"

br-equipment-control-app.code-workspace

@@ -0,0 +1,14 @@
+{
+	"folders": [
+		{
+			"path": "."
+		},
+		{
+			"path": "../pressboi"
+		},
+		{
+			"path": "../gantry"
+		}
+	],
+	"settings": {}
+}

requirements.txt

@@ -1,6 +1,7 @@
 # Runtime dependencies
 pyserial>=3.5  # For USB serial communication
 psutil>=5.9.0  # For system stats (uptime, etc.)
+jinja2>=3.0.0  # For report generation (HTML templates)
 
 # Note: Barcode/QR scanner support uses keyboard input emulation
 # Most USB/Bluetooth scanners work as HID devices, no extra dependencies needed

src/app.py

@@ -1342,7 +1342,9 @@ def process_gui_queue(self):
         except Empty:
             pass  # Queue is empty, do nothing
         finally:
-            self.root.after(100, self.process_gui_queue)
+            # Poll every 50ms for responsive DONE message handling
+            # (reduced from 100ms to minimize delay between device response and UI update)
+            self.root.after(50, self.process_gui_queue)
 
     def monitor_panel_state(self):
         """

src/device/manager.py

@@ -212,7 +212,8 @@ def get_report_handler(self, device_name, report_name):
             report_name: Name of the report (without device prefix)
             
         Returns:
-            Callable handler function or None if not found
+            tuple: (handler, error_message) - handler is the callable function or None,
+                   error_message explains why handler is None (or None if successful)
         """
         return self.registry.get_report_handler(device_name, report_name)
 

src/device/panel.py

@@ -2,6 +2,7 @@
 from tkinter import ttk
 import re
 import time
+import threading
 from src import theme
 from src.script import SCRIPT_COMMANDS
 from src.comms import devices_lock
@@ -221,6 +222,14 @@ def __init__(self, parent, script_editor_widget, device_manager, **kwargs):
         self.after(1000, self._update_connection_status)
         # Then start periodic refresh every 2 seconds
         self.after(2000, self._schedule_status_refresh)
+        
+        # Port scanning state - pre-fetch ports in background to avoid "Scanning ports..." issue
+        self._cached_serial_ports = []
+        self._port_scan_lock = threading.Lock()
+        self._port_scan_timer_id = None
+        
+        # Start background port scanner
+        self._start_port_scanner()
 
     def _on_text_scroll(self, first, last):
         """Handle scrollbar visibility based on content overflow."""
@@ -332,6 +341,74 @@ def _schedule_status_refresh(self):
         # Schedule next refresh in 2 seconds
         self.after(2000, self._schedule_status_refresh)
 
+    def _start_port_scanner(self):
+        """Start background timer to keep serial port cache fresh."""
+        def scan_ports():
+            try:
+                from src.comms import serial
+                ports = serial.list_serial_ports()
+                with self._port_scan_lock:
+                    self._cached_serial_ports = ports
+            except Exception as e:
+                print(f"[PORT_SCAN] Error scanning ports: {e}")
+            finally:
+                # Schedule next scan in 3 seconds (using main thread)
+                try:
+                    if self.winfo_exists():
+                        self._port_scan_timer_id = self.after(3000, self._schedule_port_scan)
+                except Exception:
+                    pass  # Widget might be destroyed
+        
+        # Initial scan in background thread
+        threading.Thread(target=scan_ports, daemon=True).start()
+    
+    def _schedule_port_scan(self):
+        """Schedule the next port scan in a background thread."""
+        def scan_ports():
+            try:
+                from src.comms import serial
+                ports = serial.list_serial_ports()
+                with self._port_scan_lock:
+                    self._cached_serial_ports = ports
+            except Exception as e:
+                print(f"[PORT_SCAN] Error scanning ports: {e}")
+            finally:
+                # Schedule next scan in 3 seconds (using main thread)
+                try:
+                    if self.winfo_exists():
+                        self._port_scan_timer_id = self.after(3000, self._schedule_port_scan)
+                except Exception:
+                    pass  # Widget might be destroyed
+        
+        threading.Thread(target=scan_ports, daemon=True).start()
+    
+    def _refresh_ports(self):
+        """Manually refresh port list (called from context menu)."""
+        def do_refresh():
+            try:
+                from src.comms import serial
+                ports = serial.list_serial_ports()
+                with self._port_scan_lock:
+                    self._cached_serial_ports = ports
+                # Log refresh
+                print(f"[PORT_SCAN] Manual refresh: {len(ports)} ports found")
+            except Exception as e:
+                print(f"[PORT_SCAN] Error during manual refresh: {e}")
+        
+        threading.Thread(target=do_refresh, daemon=True).start()
+    
+    def destroy(self):
+        """Clean up resources when the widget is destroyed."""
+        # Cancel the background port scanner timer
+        if hasattr(self, '_port_scan_timer_id') and self._port_scan_timer_id is not None:
+            try:
+                self.after_cancel(self._port_scan_timer_id)
+            except Exception:
+                pass  # Timer might already be cancelled
+        
+        # Call parent destroy
+        super().destroy()
+    
     def _update_connection_status(self):
         """Update only the connection status indicators without refreshing entire panel."""
         try:
@@ -1061,43 +1138,23 @@ def on_text_right_click(self, event):
                                  activebackground=theme.PRIMARY_ACCENT,
                                  activeforeground=theme.FG_COLOR)
 
-            # List available serial ports (use cached list to avoid blocking)
-            from src.comms import serial
-            # Use cached port list if available (prevents freeze when USB devices plugged/unplugged)
-            if hasattr(self, '_cached_serial_ports') and self._cached_serial_ports is not None:
-                ports = self._cached_serial_ports
-            else:
-                # First time or cache expired - show "Scanning..." and fetch in background
-                usb_submenu.add_command(label="Scanning ports...", state='disabled')
-                ports = []
-                
-                # Fetch ports in background thread to avoid blocking UI
-                def fetch_ports_async():
-                    try:
-                        import threading
-                        def get_ports():
-                            ports_list = serial.list_serial_ports()
-                            # Cache for 5 seconds
-                            self._cached_serial_ports = ports_list
-                            self._cached_ports_time = time.time()
-                            # Refresh cache every 5 seconds
-                            def clear_cache():
-                                if hasattr(self, '_cached_ports_time'):
-                                    if time.time() - self._cached_ports_time > 5:
-                                        self._cached_serial_ports = None
-                            self.after(5000, clear_cache)
-                        thread = threading.Thread(target=get_ports, daemon=True)
-                        thread.start()
-                    except Exception as e:
-                        print(f"[ERROR] Failed to fetch serial ports: {e}")
-                fetch_ports_async()
+            # Use pre-cached port list (background scanner keeps it fresh)
+            # This avoids the "Scanning ports..." race condition
+            with self._port_scan_lock:
+                ports = list(self._cached_serial_ports)  # Copy to avoid race conditions
 
             if ports:
                 for port, description in ports:
                     usb_submenu.add_command(
                         label=f"{port} - {description}",
                         command=lambda p=port, d=device_name: self.set_connection_usb(d, p)
                     )
+            else:
+                usb_submenu.add_command(label="No ports detected", state='disabled')
+            
+            # Add separator and refresh option
+            usb_submenu.add_separator()
+            usb_submenu.add_command(label="Refresh Ports", command=self._refresh_ports)
 
             connection_submenu.add_cascade(label="USB Serial", menu=usb_submenu)
 

src/device/registry.py

@@ -67,16 +67,26 @@ def _load_module_from_path(self, device_name, module_name, device_path):
     def _load_package_from_path(self, device_name, package_name, package_path):
         """
         Load a Python package (folder with __init__.py) from a specific path.
-        Returns the package module if successful, None otherwise.
+        
+        Returns:
+            tuple: (module, error_message) - module is None if loading failed,
+                   error_message is None if loading succeeded.
         """
         init_file = os.path.join(package_path, '__init__.py')
         if not os.path.exists(init_file):
-            return None
+            return None, f"Package '{package_name}' not found (missing __init__.py)"
 
         try:
             # Create a unique module name to avoid conflicts
             full_module_name = f"devices.{device_name}.{package_name}"
 
+            # Clear any cached versions of this module and its submodules
+            # This ensures we always load fresh from disk
+            modules_to_remove = [key for key in sys.modules.keys() 
+                                if key == full_module_name or key.startswith(f"{full_module_name}.")]
+            for mod_name in modules_to_remove:
+                del sys.modules[mod_name]
+            
             # Load the package from __init__.py
             spec = importlib.util.spec_from_file_location(
                 full_module_name, 
@@ -87,13 +97,15 @@ def _load_package_from_path(self, device_name, package_name, package_path):
                 module = importlib.util.module_from_spec(spec)
                 sys.modules[full_module_name] = module
                 spec.loader.exec_module(module)
-                return module
+                return module, None
         except Exception as e:
-            self.log(f"Error loading package {package_name} for {device_name}: {e}")
+            error_msg = str(e)
+            self.log(f"Error loading package {package_name} for {device_name}: {error_msg}")
             import traceback
             traceback.print_exc()
+            return None, error_msg
 
-        return None
+        return None, f"Failed to create module spec for '{package_name}'"
 
     def discover_devices(self):
         """
@@ -209,9 +221,10 @@ def _load_device_from_path(self, device_name, definition_path):
 
             # Load reports module if reports/ folder exists (as a package with __init__.py)
             reports_module = None
+            reports_load_error = None
             reports_folder = os.path.join(definition_path, 'reports')
             if os.path.isdir(reports_folder):
-                reports_module = self._load_package_from_path(device_name, 'reports', reports_folder)
+                reports_module, reports_load_error = self._load_package_from_path(device_name, 'reports', reports_folder)
 
             # Load warnings from JSON (always from definition folder)
             warnings_data = {}
@@ -242,6 +255,9 @@ def _load_device_from_path(self, device_name, definition_path):
                     'parser': parser_module,
                     'reports': reports_module,
                 },
+                'module_errors': {
+                    'reports': reports_load_error,
+                },
             }
 
             return True
@@ -463,21 +479,26 @@ def get_report_handler(self, device_name, report_name):
             report_name: Name of the report (without device prefix)
             
         Returns:
-            Callable handler function or None if not found
+            tuple: (handler, error_message) - handler is the callable function or None,
+                   error_message explains why handler is None (or None if successful)
         """
         device = self.devices.get(device_name)
         if not device:
-            return None
+            return None, f"Device '{device_name}' not found"
 
         reports_module = device.get('modules', {}).get('reports')
         if not reports_module:
-            return None
+            # Check if there was a load error
+            load_error = device.get('module_errors', {}).get('reports')
+            if load_error:
+                return None, f"Reports module failed to load: {load_error}"
+            return None, f"Reports module not available for device '{device_name}'"
 
         # Look for the handler function in the reports module
         # The handler name is typically the report_name itself (e.g., generate_press_report)
         handler = getattr(reports_module, report_name, None)
         if callable(handler):
-            return handler
+            return handler, None
 
-        return None
+        return None, f"Report handler '{report_name}' not found in reports module"
 

src/logging/data.py

@@ -7,6 +7,7 @@
 
 import csv
 import os
+import re
 import sys
 import threading
 import datetime
@@ -69,6 +70,9 @@ def _get_unique_filename(self, base_filename: str) -> str:
         # Apply job, op, and serial number formatting (handles <job>/<op>/<serial> placeholders)
         base_filename = format_filename_with_serial(base_filename, serial, job, op)
 
+        # Clean up multiple consecutive spaces (from empty template tags) and trim
+        base_filename = re.sub(r'\s+', ' ', base_filename).strip()
+        
         if not base_filename.endswith('.csv'):
             base_filename += '.csv'
 
@@ -261,6 +265,7 @@ def timer_callback():
                 freq_info = f" at {frequency} Hz" if frequency else " (synced with telemetry)"
                 log_msg = f"Started logging {var_count} variable(s){freq_info} to {os.path.basename(filepath)}"
                 log_to_terminal(log_msg, self.shared_gui_refs)
+                log_to_terminal(f"[DATA LOGGER] Full path: {filepath}", self.shared_gui_refs)
 
                 print(f"[TRACE] start_logging complete, returning success")
                 return (True, log_msg, filepath)
@@ -329,8 +334,13 @@ def _close_log_file(self, filepath: str):
         """
         log_info = self.active_logs.get(filepath)
         if log_info:
-            # Close file handle if open
+            # Close file handle if open - flush and sync to ensure Windows writes to disk
             if log_info['file_handle']:
+                try:
+                    log_info['file_handle'].flush()
+                    os.fsync(log_info['file_handle'].fileno())
+                except Exception:
+                    pass  # Ignore errors during sync (file might already be closed)
                 log_info['file_handle'].close()
 
             # Cancel timer if present