Gradual Dependency Update

.codex

@@ -0,0 +1,45 @@
+# Navigate Codex Guidance
+
+## Tkinter Threading Rules (Required)
+
+- Treat Tk widgets and Tk variables as main-thread-only resources.
+- Do not call Tk APIs from worker threads. This includes:
+  - `widget.configure(...)`, `widget.set(...)`, `widget.get(...)`
+  - `tk.Variable.set(...)`, `tk.Variable.get(...)`
+  - `root.after(...)`, `after_cancel(...)`, popup/messagebox calls
+- In this codebase, cross-thread UI access must go through:
+  - `Controller._run_on_main_thread(...)`
+  - The controller pump/drain loop: `_schedule_event_pump()` -> `_drain_main_thread_dispatch_queue()`
+
+## Approved UI Update Pattern
+
+- If work starts in a worker thread and needs UI updates:
+  - Put UI logic in a dedicated controller method.
+  - Call it with `self._run_on_main_thread(...)`.
+  - Use `wait=True` when ordering/consistency is required.
+- Keep heavy computation and I/O off the Tk thread.
+- Keep UI methods short and side-effect focused.
+
+## Controller Architecture Constraints
+
+- Do not reintroduce background-thread direct UI updates.
+- Do not start independent UI polling threads for event handling.
+- Keep event processing on Tk thread via `_schedule_event_pump()` / `update_event()`.
+- Preserve existing capture flow that dispatches UI work through:
+  - `_start_capture_ui`
+  - `_on_capture_started`
+  - `_update_capture_display`
+  - `_finish_capture_ui`
+
+## Guardrails and Diagnostics
+
+- `install_tk_thread_guard(...)` is used to detect off-main-thread Tk access.
+- If you add new UI paths, validate they do not trigger guard warnings.
+- In tests, avoid creating false cross-thread Tk usage; use controller dispatch helpers.
+
+## Practical Do/Don't
+
+- Do: `self._run_on_main_thread(self.some_ui_method, arg1, wait=True)`
+- Do: update model/data in worker thread, render in main thread.
+- Don't: call sub-controller widget methods directly from worker threads.
+- Don't: mutate Tk variables in worker threads, even if it "usually works".

.github/workflows/push_checks.yaml

@@ -7,14 +7,27 @@ on:
       - develop
   pull_request:
   workflow_dispatch:
+    inputs:
+      diagnostic_no_cov:
+        description: "Run pytest without coverage and live log CLI to diagnose teardown hangs"
+        required: false
+        default: false
+        type: boolean
 
 jobs:
   test:
     runs-on: ${{ matrix.operating-system }}
+    timeout-minutes: 45
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.9"]
-        operating-system: [windows-latest]  # [ubuntu-latest, windows-latest]
+        include:
+          - operating-system: windows-latest
+            python-version: "3.9"
+            python-tag: py39
+          - operating-system: windows-latest
+            python-version: "3.10"
+            python-tag: py310
     env:
       MPLBACKEND: Agg  # https://github.com/microsoft/azure-pipelines-tasks/issues/16426
     steps:
@@ -28,18 +41,41 @@ jobs:
           python -m pip install --upgrade pip
           pip install -e '.[dev]'
       - name: Test with pytest
+        timeout-minutes: 35
         env:
-            NAVIGATE_RUN_PLUGIN_INTEGRATION: "1"
+          NAVIGATE_RUN_PLUGIN_INTEGRATION: "1"
+          COVERAGE_FILE: .coverage.${{ matrix.python-tag }}
         run: |
-          python3 -m pytest
+          if ("${{ github.event_name }}" -eq "workflow_dispatch" -and "${{ github.event.inputs.diagnostic_no_cov }}" -eq "true") {
+            python3 -m pytest --no-cov -o log_cli=false
+          } else {
+            python3 -m pytest
+            if (Test-Path coverage.xml) {
+              Move-Item -Force coverage.xml "coverage-${{ matrix.python-tag }}.xml"
+            } else {
+              Write-Error "coverage.xml was not produced."
+            }
+          }
+      - name: Dump Python processes (Windows)
+        if: ${{ failure() && runner.os == 'Windows' }}
+        shell: pwsh
+        run: |
+          Write-Host "Tasklist snapshot for python processes:"
+          tasklist /v /fo table | Select-String -Pattern "python"
+          Write-Host "Detailed process snapshot for python executables:"
+          Get-CimInstance Win32_Process |
+            Where-Object { $_.Name -match '^python(3)?(\.exe)?$' } |
+            Select-Object ProcessId, ParentProcessId, Name, CommandLine |
+            Format-List
       - name: Codecov
+        if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.diagnostic_no_cov == 'true') }}
         uses: codecov/codecov-action@v3.1.0
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           directory: ./coverage/reports/
           env_vars: OS,PYTHON
           fail_ci_if_error: false
-          files: ./coverage.xml
-          flags: unittests
-          name: codecov-umbrella
+          files: ./coverage-${{ matrix.python-tag }}.xml
+          flags: unittests-${{ matrix.python-tag }}
+          name: codecov-${{ matrix.operating-system }}-${{ matrix.python-tag }}
           verbose: true

pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
     'PyYAML==6.0',
     'pyserial==3.5',
     'PIPython==2.6.0.1',
-    'nidaqmx==0.5.7',
+    'nidaqmx',
     'tifffile',
     'scipy',
     'pyusb==1.2.1',

pytest.ini

@@ -8,6 +8,8 @@ filterwarnings =
     ignore:'parseString' deprecated - use 'parse_string':DeprecationWarning
     ignore:'resetCache' deprecated - use 'reset_cache':DeprecationWarning
     ignore:'enablePackrat' deprecated - use 'enable_packrat':DeprecationWarning
+    ignore:Please use 'pipython\\.pidevice\\.gcserror' instead:DeprecationWarning
+    ignore::DeprecationWarning:pipython.*
 markers =
     hardware: mark tests as run on physical hardware
     integration: mark tests that rely on external integrations (network/pip)

src/navigate/controller/README.md

@@ -0,0 +1,154 @@
+# Navigate Controller Architecture
+
+This directory contains the main application controller for Navigate.
+
+The central class is `Controller` in `controller.py`. It coordinates:
+
+1. The Tkinter GUI
+2. The model process (`ObjectInSubprocess`)
+3. Sub-controllers for each major UI surface
+4. Cross-thread and cross-process event flow
+
+## Big Picture
+
+`Controller` is the orchestration layer in an MVC-style design.
+
+- View:
+  - Built from `navigate.view.*`
+  - Owned by `self.view`
+- Model:
+  - Spawned as a subprocess (`Model` or `ASIModel`)
+  - Accessed through `self.model`
+- Controller:
+  - Routes UI commands
+  - Receives model events
+  - Applies UI updates safely on the Tk main thread
+
+## Key Components
+
+- `controller.py`
+  - Main orchestration logic
+  - Command router (`execute`)
+  - Event pump and Tk thread dispatcher
+- `configuration_controller.py`
+  - Configuration mutation and validation helpers
+- `sub_controllers/`
+  - Feature-specific UI controllers (camera, stage, channels, multiposition, etc.)
+- `thread_pool.py`
+  - Resource-scoped worker thread scheduling (`model`, `camera`, `stop_stage`, ...)
+
+## Startup Flow
+
+When `Controller(...)` is constructed, it performs this sequence:
+
+1. Load and validate config files
+2. Start the model subprocess and communication channels
+3. Construct the main view and sub-controllers
+4. Install Tk thread guard (`install_tk_thread_guard`)
+5. Start the event pump (`_schedule_event_pump`)
+6. Populate experiment settings and initialize camera/mip display
+7. Show main window
+
+## Threading Model (Important)
+
+Tkinter widgets and Tk variables are main-thread-only.
+
+This codebase enforces that with:
+
+- `Controller._run_on_main_thread(...)`
+  - Enqueues UI work when called from worker threads
+  - Can wait for completion via `wait=True`
+- `_main_thread_dispatch_queue`
+  - Queue of callables to execute on Tk thread
+- `_drain_main_thread_dispatch_queue()`
+  - Executes queued UI work
+- `_schedule_event_pump()`
+  - Repeats every ~20 ms via `root.after(...)`
+  - Drains dispatch queue and model events
+
+### Rules for Contributors
+
+- Do not update Tk widgets from worker threads.
+- Do not read/write Tk variables from worker threads.
+- Do route UI work through `_run_on_main_thread(...)`.
+- Do keep non-UI heavy work in worker threads.
+
+## Event Flow: Model -> GUI
+
+Model events arrive via `self.event_queue`.
+
+`update_event()` runs on the Tk thread and handles events such as:
+
+- `warning`
+- `multiposition`
+- `update_stage`
+- `frame_rate`
+- `stop`
+- Plugin/listener events from `event_listeners`
+
+Because `update_event()` is called by `_schedule_event_pump()`, UI updates happen on the Tk main thread.
+
+## Command Flow: GUI -> Controller -> Model
+
+UI actions call `Controller.execute(command, *args)`.
+
+`execute(...)` decides whether to:
+
+- Run quickly on the current thread
+- Dispatch to worker threads (`self.threads_pool.createThread(...)`)
+- Call into model commands (`self.model.run_command(...)`)
+- Update local UI state/sub-controller state
+
+This method is the primary command routing layer and extension point for new features.
+
+## Acquisition Path
+
+Acquisition uses both worker threads and Tk-thread marshaling:
+
+1. `execute("acquire" | "autofocus" | etc.)`
+2. Worker thread calls `capture_image(...)`
+3. UI setup and updates are marshaled using:
+   - `_start_capture_ui`
+   - `_on_capture_started`
+   - `_update_capture_display`
+   - `_finish_capture_ui`
+4. Image IDs are read from `show_img_pipe` and mapped to `data_buffer`
+5. Display/histogram/progress updates happen on the Tk thread
+
+## Plugins and Additional Microscopes
+
+- Plugins can register acquisition modes and event listeners through:
+  - `add_acquisition_mode(...)`
+  - `register_event_listener(...)`
+  - `register_event_listeners(...)`
+- Additional microscope windows are launched by:
+  - `launch_additional_microscopes(...)`
+  - `destroy_virtual_microscope(...)`
+
+Even when images are consumed in worker threads, widget updates are dispatched through `_run_on_main_thread(...)`.
+
+## Safe Extension Checklist
+
+When adding new controller behavior:
+
+1. Add command handling in `execute(...)`.
+2. Keep model calls off the Tk thread when they can block.
+3. Keep all Tk widget and Tk variable access on Tk thread.
+4. If called from a worker, wrap UI calls with `_run_on_main_thread(...)`.
+5. Add tests for command routing and thread-safe UI behavior where possible.
+
+## Common Pitfalls
+
+- Direct Tk widget updates from worker threads
+  - Can trigger intermittent `TclError` or undefined behavior
+- Calling long-running model code on Tk thread
+  - Freezes the UI
+- Reintroducing ad-hoc event polling threads
+  - Bypasses the pump/drain architecture and increases race risk
+
+## Related Files
+
+- `src/navigate/controller/controller.py`
+- `src/navigate/controller/thread_pool.py`
+- `src/navigate/tools/tk_thread_guard.py`
+- `src/navigate/controller/sub_controllers/`

src/navigate/controller/controller.py

@@ -1308,8 +1308,13 @@ def execute(self, command: str, *args: Any) -> Any:
                     self.view.wait_window(feature_list_popup.popup)
 
                     # do not run acquisition if "cancel" is selected
-                    temp = self.features_popup_controller.start_acquisiton_flag
-                    delattr(self, "features_popup_controller")
+                    temp = getattr(
+                        getattr(self, "features_popup_controller", None),
+                        "start_acquisiton_flag",
+                        False,
+                    )
+                    if hasattr(self, "features_popup_controller"):
+                        delattr(self, "features_popup_controller")
                     if not temp:
                         self.set_mode_of_sub("stop")
                         return

src/navigate/controller/sub_controllers/camera_map.py

@@ -127,8 +127,8 @@ def create_camera_maps(self):
         save_path_offset = Path(save_dir).joinpath(save_name_offset)
         save_path_variance = Path(save_dir).joinpath(save_name_variance)
 
-        tifffile.imsave(save_path_offset, self.off)
-        tifffile.imsave(save_path_variance, self.var)
+        tifffile.imwrite(save_path_offset, self.off)
+        tifffile.imwrite(save_path_variance, self.var)
 
     def display_plot(self):
         """Display the offset and variance maps."""

src/navigate/controller/sub_controllers/camera_settings.py

@@ -879,11 +879,9 @@ def update_readout_time(self):
         ideally by calling a command from the camera.
         """
         sensor_mode = self.mode_widgets["Sensor"].get()
+        readout_time = self.camera_setting_dict.get("readout_time", 0)
 
-        if sensor_mode == "Normal":
-            readout_time = self.camera_setting_dict["readout_time"]
-
-        elif sensor_mode == "Light-Sheet":
+        if sensor_mode == "Light-Sheet":
             #  Progressive sensor mode operation
             readout_time = 0
 

src/navigate/controller/sub_controllers/channels_settings.py

@@ -104,12 +104,13 @@ def set_mode(self, mode="stop"):
 
         self.mode = mode
         state = "normal" if mode == "stop" else "disabled"
-        state_readonly = "readonly" if mode == "stop" else "disabled"
+        laser_state = "readonly" if mode == "stop" else "disabled"
+        filterwheel_state = "readonly"
         for i in range(self.num):
             # State set regardless of operating mode.
             self.view.channel_checks[i].config(state=state)
             self.view.interval_spins[i].config(state="disabled")
-            self.view.laser_pulldowns[i]["state"] = state_readonly
+            self.view.laser_pulldowns[i]["state"] = laser_state
 
             if self.mode != "live" or (
                 self.mode == "live" and not self.view.channel_variables[i].get()
@@ -121,7 +122,7 @@ def set_mode(self, mode="stop"):
                 for j in range(self.number_of_filter_wheels):
                     self.view.filterwheel_pulldowns[
                         i * self.number_of_filter_wheels + j
-                    ]["state"] = state_readonly
+                    ]["state"] = filterwheel_state
                 self.view.defocus_spins[i].config(state=state)
 
     def initialize(self):

src/navigate/controller/sub_controllers/features_popup.py

@@ -38,11 +38,12 @@
 import platform
 
 # Third party imports
-from PIL import Image, ImageTk
+from PIL import ImageTk
 
 # Local application imports
 from navigate.view.popups.feature_list_popup import FeatureIcon, FeatureConfigPopup
 from navigate.view.custom_widgets.ArrowLabel import ArrowLabel
+from navigate.view.theme import get_theme_color
 from navigate.controller.sub_controllers.gui import GUIController
 from navigate.tools.image import create_arrow_image
 from navigate.tools.file_functions import load_yaml_file, save_yaml_file
@@ -425,11 +426,17 @@ def draw_feature_list_graph(self, new_list_flag=True):
                         stack.append(c)
                 end_pos = c * (feature_icon_width + al_width) + feature_icon_width // 2
         if arrow_image:
-            image_gif = arrow_image.convert("P", palette=Image.ADAPTIVE)
-
             #: ImageTk.PhotoImage: The image of the feature list graph.
-            self.image = ImageTk.PhotoImage(image_gif)
-            al = tk.Label(self.feature_list_view, image=self.image)
+            self.image = ImageTk.PhotoImage(
+                arrow_image, master=self.feature_list_view
+            )
+            al = tk.Label(
+                self.feature_list_view,
+                image=self.image,
+                bg=get_theme_color("panel_bg", "#1a212b"),
+                borderwidth=0,
+                highlightthickness=0,
+            )
             al.grid(row=1, column=0, columnspan=2 * l + 1, sticky="ew")
 
     def show_config_popup(self, idx):