Merge branch 'main' into patch-6

Author: carsen-stringer

.github/workflows/test_and_deploy.yml

@@ -0,0 +1,67 @@
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: tests
+
+on: 
+  push:
+    branches:
+      - main
+    tags:
+      - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: ${{ matrix.platform }} py${{ matrix.python-version }}
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      matrix:
+        platform: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [3.8]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # these libraries, along with pytest-xvfb (added in the `deps` in tox.ini),
+      # enable testing on Qt on linux
+      - name: Install Linux libraries
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get install -y libdbus-1-3 libxkbcommon-x11-0 libxcb-icccm4 \
+            libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 \
+            libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0 pkg-config libhdf5-103 libhdf5-dev
+      # strategy borrowed from vispy for installing opengl libs on windows
+      - name: Install Windows OpenGL
+        if: runner.os == 'Windows'
+        run: |
+          git clone --depth 1 git://github.com/pyvista/gl-ci-helpers.git
+          powershell gl-ci-helpers/appveyor/install_opengl.ps1
+      # note: if you need dependencies from conda, considering using
+      # setup-miniconda: https://github.com/conda-incubator/setup-miniconda
+      # and
+      # tox-conda: https://github.com/tox-dev/tox-conda
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install wheel setuptools tox tox-gh-actions
+          pip install dvc==1.11.0 pydrive2
+      - name: Get test data
+        run: |
+          dvc pull -r gdrive-travis
+      # this runs the platform-specific tests declared in tox.ini
+      - name: Test with tox
+        run: tox
+        env:
+          PLATFORM: ${{ matrix.platform }}
+
+      - name: Coverage
+        uses: codecov/codecov-action@v1

README.md

@@ -1,3 +1,5 @@
+Note: Marius and Carsen are currently at The Neuromatch Academy, and we won't do anything Suite2p-related until after the deep learning course!
+
 # suite2p <img src="suite2p/logo/logo_unshaded.png" width="250" title="sweet two pea" alt="sweet two pea" align="right" vspace = "50">
 
 [![Documentation Status](https://readthedocs.org/projects/suite2p/badge/?version=dev)](https://suite2p.readthedocs.io/en/dev/?badge=dev)

benchmarks/registration_metrics.py

@@ -25,6 +25,8 @@ def registration_metrics(data_path, tiff_list, ops, nPC=10):
         ops['tiff_list'] = tiff_list
 
     result_ops = suite2p.run_s2p(ops)
+    if not isinstance(result_ops, list) or not isinstance(result_ops, np.ndarray):
+        result_ops = [result_ops]
     metric_results = []
     for nplane, result_op in enumerate(result_ops):
         offsets = result_op['regDX']

docs/settings.rst

@@ -104,7 +104,7 @@ Registration
   of the gaussian used to smooth the data before phase correlation is computed.
   Might need this to be set to 1 or 2 for low SNR data.
 
-- **keep_movie_raw**: (*bool, default: True*) whether or not to keep
+- **keep_movie_raw**: (*bool, default: False*) whether or not to keep
   the binary file of the non-registered frames. You can view the
   registered and non-registered binaries together in the GUI in the
   "View registered binaries" view if you set this to *True*.

environment.yml

@@ -1,27 +1,29 @@
 name: suite2p
 channels:
   - conda-forge
+  - pytorch
 dependencies:
-  - python>=3.7.6,<3.8
+  - python>=3.8,<3.9
   - pip
-  - mkl_fft=1.0.10
-  - mkl=2019.3
+  - mkl
   - tbb
-  - numpy<1.19.0
-  - numba<0.50.0
+  - numpy
+  - numba
   - matplotlib
   - scipy
   - scikit-learn
-  - pyqt
+  - pytorch>=1.7.1
+  - h5py=2.10.0
   - pip:
-    - h5py==2.10.0
+    - pyqt5
+    - pyqt5.sip
     - natsort
     - rastermap>0.1.0
     - tifffile
     - scanimage-tiff-reader>=1.4.1
     - pyqtgraph
     - importlib-metadata
     - paramiko
-    - pynwb<1.4.0
+    - pynwb
     - sbxreader
     - suite2p

setup.py

@@ -1,5 +1,50 @@
 import setuptools
 
+install_deps = ['importlib-metadata',
+        'natsort',
+        'rastermap>0.1.0',
+        'tifffile',
+        'scanimage-tiff-reader>=1.4.1',
+        'torch>=1.7.1',
+        'paramiko',
+        'numpy>=1.16',
+        'numba>=0.43.1',
+        'matplotlib',
+        'scipy',
+        'h5py',
+        'sbxreader',
+        'scikit-learn',]
+
+gui_deps = [
+        "pyqt5",
+        "pyqt5-tools",
+        "pyqt5.sip",
+        'pyqtgraph',
+        'rastermap>0.1.0',
+      ]
+data_deps = [
+        "dvc==1.11.0",
+        "pydrive2",
+      ]
+nwb_deps = [
+        "pynwb",
+      ]
+test_deps = [
+      'pytest',
+      'pytest-qt==3.3.0',
+    ]
+
+all_deps = gui_deps + data_deps + nwb_deps
+
+try:
+    import torch
+    a = torch.ones(2, 3)
+    version = int(torch.__version__[2])
+    if version >= 6:
+        install_deps.remove('torch')
+except:
+    pass
+
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
@@ -17,24 +62,8 @@
       'setuptools_scm',
     ],
     use_scm_version=True,
-    install_requires=['importlib-metadata',
-        'natsort',
-        'rastermap>0.1.0',
-        'tifffile',
-        'scanimage-tiff-reader>=1.4.1',
-        'pyqtgraph',
-        'paramiko',
-        'numpy>=1.16',
-        'numba>=0.43.1',
-        'matplotlib',
-        'scipy',
-        'h5py',
-        'sbxreader',
-        'scikit-learn',],  # see environment.yml for this info.
-    tests_require=[
-      'pytest',
-      'pytest-qt',
-    ],
+    install_requires=install_deps,
+    tests_require=test_deps,
     extras_require={
       "docs": [
         'sphinx>=3.0',
@@ -43,24 +72,10 @@
         'sphinx-prompt',
         'sphinx-autodoc-typehints',
       ],
-      # Note: Available in pypi, but cleaner to install as pyqt from conda.
-      "gui": [
-        "pyqt5",
-        "pyqt5-tools",
-        "pyqt5.sip",
-      ],
-      # Note: Not currently available in pip: use conda to install.
-      "mkl": [
-        "mkl_fft>=1.0.10",
-        "mkl>=2019.3",
-      ],
-      "data": [
-        "dvc>=1.1",
-        "pydrive2",
-      ],
-      "nwb": [
-        "pynwb<1.4.0",
-      ]
+      "gui": gui_deps,
+      "data": data_deps,
+      "nwb": nwb_deps,
+      "all": all_deps
     },
     include_package_data=True,
     classifiers=[

suite2p/__init__.py

@@ -1,6 +1,6 @@
 from .run_s2p import run_s2p, default_ops
-from .gui import run as run_gui
 from .detection import ROI
+from .version import version
 
 
 name = "suite2p"

suite2p/__main__.py

@@ -9,7 +9,7 @@ def add_args(parser: argparse.ArgumentParser):
     parser.add_argument('--single_plane', action='store_true', help='run single plane ops')
     parser.add_argument('--ops', default=[], type=str, help='options')
     parser.add_argument('--db', default=[], type=str, help='options')
-    #parser.add_argument('--version', action='store_true', help='print version number.')
+    parser.add_argument('--version', action='store_true', help='print version number.')
     ops0 = default_ops()
     for k in ops0.keys():
         v = dict(default=ops0[k], help='{0} : {1}'.format(k, ops0[k]))
@@ -59,7 +59,7 @@ def parse_args(parser: argparse.ArgumentParser):
 
 def main():
     args, ops = parse_args(add_args(argparse.ArgumentParser(description='Suite2p parameters')))
-    if 0:#args.version:
+    if args.version:
         print("suite2p v{}".format(version))
     elif args.single_plane and args.ops:
         from suite2p.run_s2p import run_plane

suite2p/detection/detect.py

@@ -47,7 +47,7 @@ def detect(ops, classfile=None):
 
     if ops.get('anatomical_only', 0) > 0 and CELLPOSE_INSTALLED:
         print('>>>> CELLPOSE finding masks in ' + ['max_proj / mean_img', 'mean_img', 'enhanced_mean_img'][int(ops['anatomical_only'])-1])
-        stats = anatomical.select_rois(
+        stat = anatomical.select_rois(
                     ops=ops,
                     mov=mov,
                     dy=dy,
@@ -57,7 +57,7 @@ def detect(ops, classfile=None):
                     diameter=ops['diameter'])
 
     else:            
-        stats = select_rois(
+        stat = select_rois(
             ops=ops,
             mov=mov,
             dy=dy,
@@ -74,10 +74,10 @@ def detect(ops, classfile=None):
     if 'meanImg_chan2' in ops:
         if 'chan2_thres' not in ops:
             ops['chan2_thres'] = 0.65
-        ops, redcell = chan2detect.detect(ops, stats)
+        ops, redcell = chan2detect.detect(ops, stat)
         np.save(Path(ops['save_path']).joinpath('redcell.npy'), redcell)
 
-    return ops, stats
+    return ops, stat
 
 def select_rois(ops: Dict[str, Any], mov: np.ndarray, dy: int, dx: int, Ly: int, Lx: int, 
                 max_overlap: float = True, sparse_mode: bool = True, do_crop: bool=True,
@@ -86,7 +86,7 @@ def select_rois(ops: Dict[str, Any], mov: np.ndarray, dy: int, dx: int, Ly: int,
     t0 = time.time()
     if sparse_mode:
         ops.update({'Lyc': mov.shape[1], 'Lxc': mov.shape[2]})
-        new_ops, stats = sparsedetect.sparsery(
+        new_ops, stat = sparsedetect.sparsery(
             mov=mov,
             high_pass=ops['high_pass'],
             neuropil_high_pass=ops['spatial_hp_detect'],
@@ -100,29 +100,32 @@ def select_rois(ops: Dict[str, Any], mov: np.ndarray, dy: int, dx: int, Ly: int,
         )
         ops.update(new_ops)
     else:
-        ops, stats = sourcery.sourcery(mov=mov, ops=ops)
+        ops, stat = sourcery.sourcery(mov=mov, ops=ops)
 
-    print('Detected %d ROIs, %0.2f sec' % (len(stats), time.time() - t0))
-    stats = np.array(stats)
+    print('Detected %d ROIs, %0.2f sec' % (len(stat), time.time() - t0))
+    stat = np.array(stat)
+    
+    if len(stat)==0:
+        raise ValueError("no ROIs were found -- check registered binary and maybe change spatial scale")
 
     if ops['preclassify'] > 0:
         if classfile is None:
             print(f'NOTE: Applying user classifier at {str(user_classfile)}')
             classfile = user_classfile
 
-        stats =  roi_stats(stats, dy, dx, Ly, Lx, do_crop=do_crop)
-        if len(stats) == 0:
+        stat =  roi_stats(stat, dy, dx, Ly, Lx, do_crop=do_crop)
+        if len(stat) == 0:
             iscell = np.zeros((0, 2))
         else:
-            iscell = classify(stat=stats, classfile=classfile)
+            iscell = classify(stat=stat, classfile=classfile)
         np.save(Path(ops['save_path']).joinpath('iscell.npy'), iscell)
         ic = (iscell[:,0]>ops['preclassify']).flatten().astype(np.bool)
-        stats = stats[ic]
+        stat = stat[ic]
         print('Preclassify threshold %0.2f, %d ROIs removed' % (ops['preclassify'], (~ic).sum()))
-        
-    # add ROI stats to stats
-    stats = roi_stats(stats, dy, dx, Ly, Lx, max_overlap=max_overlap, do_crop=do_crop)
+            
+    # add ROI stat to stat
+    stat = roi_stats(stat, dy, dx, Ly, Lx, max_overlap=max_overlap, do_crop=do_crop)
 
-    print('After removing overlaps, %d ROIs remain' % (len(stats)))
-    return stats
+    print('After removing overlaps, %d ROIs remain' % (len(stat)))
+    return stat
 

suite2p/detection/sparsedetect.py

@@ -157,7 +157,8 @@ def extendROI(ypix, xpix, Ly, Lx,niter=1):
         ypix,xpix = yu[:, ix]
     return ypix,xpix
 
-def two_comps(mpix0, lam, Th2):
+
+def two_comps(mpix0,  lam, Th2):
     """ check if splitting ROI increases variance explained
 
     Parameters