Merge branch 'main' into fix-skip-neuropil

Author: carsen-stringer

.github/workflows/test_and_deploy.yml

@@ -0,0 +1,67 @@
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: tests
+
+on: 
+  push:
+    branches:
+      - main
+    tags:
+      - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: ${{ matrix.platform }} py${{ matrix.python-version }}
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      matrix:
+        platform: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [3.8]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # these libraries, along with pytest-xvfb (added in the `deps` in tox.ini),
+      # enable testing on Qt on linux
+      - name: Install Linux libraries
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get install -y libdbus-1-3 libxkbcommon-x11-0 libxcb-icccm4 \
+            libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 \
+            libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0 pkg-config libhdf5-103 libhdf5-dev
+      # strategy borrowed from vispy for installing opengl libs on windows
+      - name: Install Windows OpenGL
+        if: runner.os == 'Windows'
+        run: |
+          git clone --depth 1 git://github.com/pyvista/gl-ci-helpers.git
+          powershell gl-ci-helpers/appveyor/install_opengl.ps1
+      # note: if you need dependencies from conda, considering using
+      # setup-miniconda: https://github.com/conda-incubator/setup-miniconda
+      # and
+      # tox-conda: https://github.com/tox-dev/tox-conda
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install wheel setuptools tox tox-gh-actions
+          pip install dvc==1.11.0 pydrive2
+      - name: Get test data
+        run: |
+          dvc pull -r gdrive-travis
+      # this runs the platform-specific tests declared in tox.ini
+      - name: Test with tox
+        run: tox
+        env:
+          PLATFORM: ${{ matrix.platform }}
+
+      - name: Coverage
+        uses: codecov/codecov-action@v1

README.md

@@ -1,3 +1,5 @@
+Note: Marius and Carsen are currently at The Neuromatch Academy, and we won't do anything Suite2p-related until after the deep learning course!
+
 # suite2p <img src="suite2p/logo/logo_unshaded.png" width="250" title="sweet two pea" alt="sweet two pea" align="right" vspace = "50">
 
 [![Documentation Status](https://readthedocs.org/projects/suite2p/badge/?version=dev)](https://suite2p.readthedocs.io/en/dev/?badge=dev)

benchmarks/registration_metrics.py

@@ -25,6 +25,8 @@ def registration_metrics(data_path, tiff_list, ops, nPC=10):
         ops['tiff_list'] = tiff_list
 
     result_ops = suite2p.run_s2p(ops)
+    if not isinstance(result_ops, list) or not isinstance(result_ops, np.ndarray):
+        result_ops = [result_ops]
     metric_results = []
     for nplane, result_op in enumerate(result_ops):
         offsets = result_op['regDX']

environment.yml

@@ -1,20 +1,22 @@
 name: suite2p
 channels:
   - conda-forge
+  - pytorch
 dependencies:
-  - python>=3.7.6,<3.8
+  - python>=3.8,<3.9
   - pip
-  - mkl_fft=1.0.10
-  - mkl=2019.3
+  - mkl
   - tbb
-  - numpy<1.19.0
-  - numba<0.50.0
+  - numpy
+  - numba
   - matplotlib
   - scipy
   - scikit-learn
-  - pyqt
+  - pytorch>=1.7.1
+  - h5py=2.10.0
   - pip:
-    - h5py==2.10.0
+    - pyqt5
+    - pyqt5.sip
     - natsort
     - rastermap>0.1.0
     - tifffile

setup.py

@@ -1,5 +1,50 @@
 import setuptools
 
+install_deps = ['importlib-metadata',
+        'natsort',
+        'rastermap>0.1.0',
+        'tifffile',
+        'scanimage-tiff-reader>=1.4.1',
+        'torch>=1.7.1',
+        'paramiko',
+        'numpy>=1.16',
+        'numba>=0.43.1',
+        'matplotlib',
+        'scipy',
+        'h5py',
+        'sbxreader',
+        'scikit-learn',]
+
+gui_deps = [
+        "pyqt5",
+        "pyqt5-tools",
+        "pyqt5.sip",
+        'pyqtgraph',
+        'rastermap>0.1.0',
+      ]
+data_deps = [
+        "dvc==1.11.0",
+        "pydrive2",
+      ]
+nwb_deps = [
+        "pynwb",
+      ]
+test_deps = [
+      'pytest',
+      'pytest-qt==3.3.0',
+    ]
+
+all_deps = gui_deps + data_deps + nwb_deps
+
+try:
+    import torch
+    a = torch.ones(2, 3)
+    version = int(torch.__version__[2])
+    if version >= 6:
+        install_deps.remove('torch')
+except:
+    pass
+
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
@@ -17,24 +62,8 @@
       'setuptools_scm',
     ],
     use_scm_version=True,
-    install_requires=['importlib-metadata',
-        'natsort',
-        'rastermap>0.1.0',
-        'tifffile',
-        'scanimage-tiff-reader>=1.4.1',
-        'pyqtgraph',
-        'paramiko',
-        'numpy>=1.16',
-        'numba>=0.43.1',
-        'matplotlib',
-        'scipy',
-        'h5py',
-        'sbxreader',
-        'scikit-learn',],  # see environment.yml for this info.
-    tests_require=[
-      'pytest',
-      'pytest-qt',
-    ],
+    install_requires=install_deps,
+    tests_require=test_deps,
     extras_require={
       "docs": [
         'sphinx>=3.0',
@@ -43,24 +72,10 @@
         'sphinx-prompt',
         'sphinx-autodoc-typehints',
       ],
-      # Note: Available in pypi, but cleaner to install as pyqt from conda.
-      "gui": [
-        "pyqt5",
-        "pyqt5-tools",
-        "pyqt5.sip",
-      ],
-      # Note: Not currently available in pip: use conda to install.
-      "mkl": [
-        "mkl_fft>=1.0.10",
-        "mkl>=2019.3",
-      ],
-      "data": [
-        "dvc==1.11.0",
-        "pydrive2",
-      ],
-      "nwb": [
-        "pynwb",
-      ]
+      "gui": gui_deps,
+      "data": data_deps,
+      "nwb": nwb_deps,
+      "all": all_deps
     },
     include_package_data=True,
     classifiers=[

suite2p/__init__.py

@@ -1,6 +1,6 @@
 from .run_s2p import run_s2p, default_ops
-from .gui import run as run_gui
 from .detection import ROI
+from .version import version
 
 
 name = "suite2p"

suite2p/__main__.py

@@ -9,7 +9,7 @@ def add_args(parser: argparse.ArgumentParser):
     parser.add_argument('--single_plane', action='store_true', help='run single plane ops')
     parser.add_argument('--ops', default=[], type=str, help='options')
     parser.add_argument('--db', default=[], type=str, help='options')
-    #parser.add_argument('--version', action='store_true', help='print version number.')
+    parser.add_argument('--version', action='store_true', help='print version number.')
     ops0 = default_ops()
     for k in ops0.keys():
         v = dict(default=ops0[k], help='{0} : {1}'.format(k, ops0[k]))
@@ -59,7 +59,7 @@ def parse_args(parser: argparse.ArgumentParser):
 
 def main():
     args, ops = parse_args(add_args(argparse.ArgumentParser(description='Suite2p parameters')))
-    if 0:#args.version:
+    if args.version:
         print("suite2p v{}".format(version))
     elif args.single_plane and args.ops:
         from suite2p.run_s2p import run_plane

suite2p/detection/sparsedetect.py

@@ -157,7 +157,8 @@ def extendROI(ypix, xpix, Ly, Lx,niter=1):
         ypix,xpix = yu[:, ix]
     return ypix,xpix
 
-def two_comps(mpix0, lam, Th2):
+
+def two_comps(mpix0,  lam, Th2):
     """ check if splitting ROI increases variance explained
 
     Parameters

suite2p/extraction/dcnv.py

@@ -82,7 +82,7 @@ def oasis(F: np.ndarray, batch_size: int, tau: float, fs: float) -> np.ndarray:
 
 
 def preprocess(F: np.ndarray, baseline: str, win_baseline: float,
-               sig_baseline: float, fs: float, prctile_baseline: float = 0.9) -> np.ndarray:
+               sig_baseline: float, fs: float, prctile_baseline: float = 8) -> np.ndarray:
     """ preprocesses fluorescence traces for spike deconvolution
 
     baseline-subtraction with window 'win_baseline'

suite2p/extraction/extract.py

@@ -6,6 +6,7 @@
 from numba.typed import List
 from scipy import stats, signal
 from .masks import create_masks
+from ..io import BinaryFile
 
 def extract_traces(ops, cell_masks, neuropil_masks, reg_file):
     """ extracts activity from reg_file using masks in stat and neuropil_masks
@@ -33,8 +34,9 @@ def extract_traces(ops, cell_masks, neuropil_masks, reg_file):
         each element is neuropil pixels in (Ly*Lx) coordinates
         GOING TO BE DEPRECATED: size [ncells x npixels] where weights of each mask are elements
 
-    reg_file : string
-        path to registered binary file
+    reg_file : io.BinaryFile object
+        io.BinaryFile object that has iter_frames(batch_size=ops['batch_size']) method
+        
 
     Returns
     ----------------
@@ -58,36 +60,30 @@ def extract_traces(ops, cell_masks, neuropil_masks, reg_file):
     F    = np.zeros((ncells, nframes),np.float32)
     Fneu = np.zeros((ncells, nframes),np.float32)
 
-    reg_file = open(reg_file, 'rb')
     nimgbatch = int(nimgbatch)
-    block_size = Ly*Lx*nimgbatch*2
-    ix = 0
-    data = 1
-    neuropil_ipix = None
+    
+    cell_ipix = [cell_mask[0].astype(np.int64) for cell_mask in cell_masks]
+    cell_lam = [cell_mask[1].astype(np.float32) for cell_mask in cell_masks]
 
-    cell_ipix = List()
-    [cell_ipix.append(cell_mask[0].astype(np.int64)) for cell_mask in cell_masks]
-    cell_lam = List()
-    [cell_lam.append(cell_mask[1].astype(np.float32)) for cell_mask in cell_masks]
     if neuropil_masks is not None:
         if isinstance(neuropil_masks, np.ndarray) and neuropil_masks.shape[1] == Ly*Lx:
             neuropil_masks = [np.nonzero(neuropil_mask)[0] for neuropil_mask in neuropil_masks]
         else:
             neuropil_masks = [neuropil_mask.astype(np.int64) for neuropil_mask in neuropil_masks]
-        neuropil_ipix = List()
-        [neuropil_ipix.append(neuropil_mask) for neuropil_mask in neuropil_masks]
+        neuropil_ipix = neuropil_masks
         neuropil_npix = np.array([len(neuropil_ipixi) for neuropil_ipixi in neuropil_ipix]).astype(np.float32)
+    else:
+        neuropil_ipix = None
 
-    while data is not None:
-        buff = reg_file.read(block_size)
-        data = np.frombuffer(buff, dtype=np.int16, offset=0)
-        nimg = int(np.floor(data.size / (Ly*Lx)))
+    ix = 0
+    for k, (_, data) in enumerate(reg_file.iter_frames(batch_size=ops['batch_size'])):
+        nimg = data.shape[0]
         if nimg == 0:
             break
-        data = np.reshape(data, (-1, Ly, Lx))
         inds = ix+np.arange(0,nimg,1,int)
         data = np.reshape(data, (nimg,-1)).astype(np.float32)
         Fi = np.zeros((ncells, data.shape[0]), np.float32)
+        
         # extract traces and neuropil
 
         # (WITHOUT NUMBA)
@@ -127,9 +123,13 @@ def extract_traces_from_masks(ops, cell_masks, neuropil_masks):
     
     """
     F_chan2, Fneu_chan2 = [], []
-    F, Fneu, ops = extract_traces(ops, cell_masks, neuropil_masks, ops['reg_file'])
+    with BinaryFile(Ly=ops['Ly'], Lx=ops['Lx'],
+                    read_filename=ops['reg_file']) as f:    
+        F, Fneu, ops = extract_traces(ops, cell_masks, neuropil_masks, f)
     if 'reg_file_chan2' in ops:
-        F_chan2, Fneu_chan2, _ = extract_traces(ops.copy(), cell_masks, neuropil_masks, ops['reg_file_chan2'])
+        with BinaryFile(Ly=ops['Ly'], Lx=ops['Lx'],
+                        read_filename=ops['reg_file_chan2']) as f:    
+            F_chan2, Fneu_chan2, _ = extract_traces(ops.copy(), cell_masks, neuropil_masks, f)
     return F, Fneu, F_chan2, Fneu_chan2, ops
 
 def create_masks_and_extract(ops, stat, cell_masks=None, neuropil_masks=None):