Merge pull request #910 from MouseLand/refactor

Fixes bugs with badframes and nonrigid registration.

Author: chriski777

.github/workflows/test_and_deploy.yml

@@ -32,11 +32,13 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+
       # these libraries, along with pytest-xvfb (added in the `deps` in tox.ini),
       # enable testing on Qt on linux
       - name: Install Linux libraries
         if: runner.os == 'Linux'
         run: |
+          sudo apt-get update
           sudo apt-get install -y libdbus-1-3 libxkbcommon-x11-0 libxcb-icccm4 \
             libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 \
             libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0 pkg-config libhdf5-103 libhdf5-dev
@@ -54,25 +56,24 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install wheel setuptools tox tox-gh-actions
-          pip install dvc==1.11.0 pydrive2
-      # For debugging purposes, allows one to ssh into host machine. 
-      # Follow instructions in  https://docs.github.com/en/authentication/connecting-to-github-with-ssh/adding-a-new-ssh-key-to-your-github-account 
-      # to add your ssh keys.
-      # MAKE SURE TO COMMENT OUT IF NOT DEBUGGING!
-#       - name: Setup upterm session 
-#         if: runner.os == 'macOS'
-#         uses: lhotari/action-upterm@v1
-#         with:
-#           ## limits ssh access and adds the ssh public key for the user which triggered the workflow
-#           limit-access-to-actor: true
-#           ## limits ssh access and adds the ssh public keys of the listed GitHub users
-#           limit-access-to-users: chriski777, carsen-stringer
+          pip install pydrive2 py # Added py due to pytest tox issues requiring py module.
 
       - name: Test with tox
         run: tox
         env:
-          PLATFORM: ${{ matrix.platform }}
-
+          PLATFORM: ${{ matrix.platform }}    
+      # ONLY UNCOMMENT SECTION BELOW FOR DEBUGGING PURPOSES: allows one to ssh into host machine. 
+      # Follow instructions in  https://docs.github.com/en/authentication/connecting-to-github-with-ssh/adding-a-new-ssh-key-to-your-github-account 
+      # to add your ssh keys.
+#      - name: Job failed. Activating debugging mode via up-term.
+#        if: ${{ failure() }}
+#        uses: lhotari/action-upterm@v1
+#        with:
+#           ## limits ssh access and adds the ssh public key for the user which triggered the workflow
+#            limit-access-to-actor: true
+#           ## limits ssh access and adds the ssh public keys of the listed GitHub users
+#            limit-access-to-users: chriski777, carsen-stringera
+            
       - name: Coverage
         # Only run coverage once
         if: runner.os == 'Linux'

README.md

@@ -62,10 +62,11 @@ pip install --upgrade suite2p
 ~~~~
 
 ### Installation for Macs with Apple Silicon chips (e.g., M1)
-1. Set up a Rosetta terminal following step 1 in this [link](https://dev.to/courier/tips-and-tricks-to-setup-your-apple-m1-for-development-547g). 
-2. Open up the newly created Rosetta terminal and follow steps 1 & 2 in the installation section [above](#installation_section) to install anaconda.
-3. Use the following command `CONDA_SUBDIR=osx-64 conda create --name suite2p python=3.8`
-4. Follow steps 4-7 in the installation section [above](#installation_section) to install the `suite2p` package. 
+1. Download an iTerm2 terminal from this [link](https://iterm2.com/). Install it into your /Applications folder. If you already have downloaded iTerm, duplicate it and give it whatever name you'd like (e.g., "iterm2Rosetta"). 
+2. Navigate to the iTerm app you will use, right click it, and then select "Get Info". Check "Open using Rosetta". 
+3. Open up this iTerm app and follow steps 1 & 2 in the installation section [above](#installation_section) to install anaconda.
+4. Use the following command `CONDA_SUBDIR=osx-64 conda create --name suite2p python=3.8`
+5. Follow steps 4-7 in the installation section [above](#installation_section) to install the `suite2p` package. 
 
 
 ### Installing the latest github version of the code

suite2p/detection/detect.py

@@ -35,9 +35,10 @@ def bin_movie(f_reg, bin_size, yrange=None, xrange=None, badframes=None):
 	batch_size = min(good_frames.sum(), 500)
 	Lyc = yrange[1] - yrange[0]
 	Lxc = xrange[1] - xrange[0]
-	mov = np.zeros((n_frames//bin_size, Lyc, Lxc), np.float32)
+	# Need to adjust binned movie size after calculating bad frames
+	num_good_frames = good_frames.sum()
+	mov = np.zeros((num_good_frames//bin_size, Lyc, Lxc), np.float32)
 	ik = 0
-	
 	t0 = time.time()
 	for k in np.arange(0, n_frames, batch_size):
 		data = f_reg[k : min(k + batch_size, n_frames)]
@@ -61,7 +62,6 @@ def bin_movie(f_reg, bin_size, yrange=None, xrange=None, badframes=None):
 		ik += n_bins
 
 	print('Binned movie of size [%d,%d,%d] created in %0.2f sec.' % (mov.shape[0], mov.shape[1], mov.shape[2], time.time() - t0))
-
 	return mov
 
 

suite2p/io/nwb.py

@@ -359,14 +359,14 @@ def save_nwb(save_folder):
             grid_spacing=([2.0, 2.0, 30.0] if multiplane else [2.0, 2.0]),
             grid_spacing_unit="microns",
         )
-
         # link to external data
+        external_data = ops["filelist"] if "filelist" in ops else [""]
         image_series = TwoPhotonSeries(
             name="TwoPhotonSeries",
             dimension=[ops["Ly"], ops["Lx"]],
-            external_file=(ops["filelist"] if "filelist" in ops else [""]),
+            external_file=external_data,
             imaging_plane=imaging_plane,
-            starting_frame=[0],
+            starting_frame=[0 for i in range(len(external_data))],
             format="external",
             starting_time=0.0,
             rate=ops["fs"] * ops["nplanes"],

suite2p/registration/register.py

@@ -179,6 +179,7 @@ def compute_reference_masks(refImg, ops=default_ops()):
             smooth_sigma=ops['smooth_sigma'],
         )
         Ly, Lx = refImg.shape
+        blocks = []
         if ops.get('nonrigid'):
             blocks = nonrigid.make_blocks(Ly=Ly, Lx=Lx, block_size=ops['block_size'])
 
@@ -190,7 +191,7 @@ def compute_reference_masks(refImg, ops=default_ops()):
                 xblock=blocks[1],
             )
         else:
-            maskMulNR, maskOffsetNR, cfRefImgNR, blocks = [], [], [], []
+            maskMulNR, maskOffsetNR, cfRefImgNR = [], [], [] 
 
         return maskMul, maskOffset, cfRefImg, maskMulNR, maskOffsetNR, cfRefImgNR, blocks
 
@@ -464,7 +465,8 @@ def shift_frames_and_write(f_alt_in, f_alt_out=None, yoff=None, xoff=None, yoff1
         raise ValueError('no rigid registration offsets provided')
     elif yoff.shape[0] != n_frames or xoff.shape[0] != n_frames:
         raise ValueError('rigid registration offsets are not the same size as input frames')
-
+    # Overwrite blocks if nonrigid registration is activated   
+    blocks = None
     if ops.get('nonrigid'):
         if yoff1 is None or xoff1 is None:
             raise ValueError('nonrigid registration is activated but no nonrigid shifts provided')
@@ -636,16 +638,17 @@ def registration_wrapper(f_reg, f_raw=None, f_reg_chan2=None, f_raw_chan2=None,
 
 
     # compute valid region
-    # ignore user-specified bad_frames.npy
     badframes = np.zeros(n_frames, 'bool')
     if 'data_path' in ops and len(ops['data_path']) > 0:
         badfrfile = path.abspath(path.join(ops['data_path'][0], 'bad_frames.npy'))
+        # Check if badframes file exists
         if path.isfile(badfrfile):
             print('bad frames file path: %s'%badfrfile)
-            badframes = np.load(badfrfile)
-            badframes = badframes.flatten().astype(int)
-            badframes = True
-            print('number of badframes: %d'%ops['badframes'].sum())
+            bf_indices = np.load(badfrfile)
+            bf_indices = bf_indices.flatten().astype(int)
+            # Set indices of badframes to true
+            badframes[bf_indices] = True
+            print('number of badframes: %d'%badframes.sum())
 
     # return frames which fall outside range
     badframes, yrange, xrange = compute_crop(

tox.ini

@@ -21,13 +21,14 @@ platform =
 passenv = 
     CI
     GITHUB_ACTIONS
-    DISPLAY XAUTHORITY
+    DISPLAY,XAUTHORITY
     NUMPY_EXPERIMENTAL_ARRAY_FUNCTION
     PYVISTA_OFF_SCREEN
 extras = all
 deps = 
     .[all]
+    py # Needed for py-test import error
     pytest  # https://docs.pytest.org/en/latest/contents.html
     pytest-cov  # https://pytest-cov.readthedocs.io/en/latest/
     pytest-xvfb ; sys_platform == 'linux'
-commands = pytest -v --color=yes --cov=suite2p --cov-report=xml
+commands = pytest -v --color=yes --cov=suite2p --cov-report=xml