Merge pull request #158 from kabilar/main

Update `get_calcium_imaging_files` and installation instructions

Author: Thinh Nguyen

.devcontainer/Dockerfile

@@ -22,7 +22,7 @@ RUN \
     apt-get install sudo git bash-completion graphviz default-mysql-client s3fs procps -y && \
     usermod -aG sudo vscode && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
-    pip install --no-cache-dir --upgrade black pip && \
+    pip install --no-cache-dir --upgrade black pip nbconvert && \
     echo '. /etc/bash_completion' >> /home/vscode/.bashrc && \
     echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /home/vscode/.bashrc && \
     apt-get clean
@@ -32,8 +32,9 @@ COPY ./ /tmp/element-calcium-imaging/
 RUN \
     # pipeline dependencies
     apt-get install gcc g++ ffmpeg libsm6 libxext6 -y && \
-    pip install --no-cache-dir -e /tmp/element-calcium-imaging[caiman_requirements] && \
-    pip install --no-cache-dir -e /tmp/element-calcium-imaging[caiman,elements,nd2,sbxreader,scanreader,suite2p,tests] && \
+    # pip install --no-cache-dir -e /tmp/element-calcium-imaging[caiman_requirements] && \
+    # pip install --no-cache-dir -e /tmp/element-calcium-imaging[caiman] && \
+    pip install --no-cache-dir -e /tmp/element-calcium-imaging[elements,nd2,sbxreader,scanreader,suite2p,tests] && \
     # clean up
     rm -rf /tmp/element-calcium-imaging && \
     apt-get clean

CHANGELOG.md

@@ -3,6 +3,12 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.8.0] - 2023-08-08
+
++ Update - Rename `get_image_files` to `get_calcium_imaging_files`, and update arguments
++ Update - Remove CaImAn installation from the Dev Container due to memory limits of a free GitHub Codespace account
++ Update - Installation instructions
+
 ## [0.7.9] - 2023-07-27
 
 + Update - GitHub Actions to run every Monday at 8am
@@ -167,6 +173,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 + Add - `scan` and `imaging` modules
 + Add - Readers for `ScanImage`, `ScanBox`, `Suite2p`, `CaImAn`
 
+[0.8.0]: https://github.com/datajoint/element-calcium-imaging/releases/tag/0.8.0
 [0.7.9]: https://github.com/datajoint/element-calcium-imaging/releases/tag/0.7.9
 [0.7.8]: https://github.com/datajoint/element-calcium-imaging/releases/tag/0.7.8
 [0.7.7]: https://github.com/datajoint/element-calcium-imaging/releases/tag/0.7.7

README.md

@@ -1,5 +1,3 @@
-[![PyPI version](https://badge.fury.io/py/element-calcium-imaging.svg)](http://badge.fury.io/py/element-calcium-imaging)
-
 # DataJoint Element for Functional Calcium Imaging
 
 DataJoint Element for functional calcium imaging with 
@@ -32,11 +30,19 @@ documentation page.
 
 ## Getting Started
 
-+ Install from PyPI
++ Please fork this repository
+
++ Clone the repository to your computer
+
+  ```bash
+  git clone https://github.com/<enter_github_username>/element-calcium-imaging
+  ```
+
++ Install with `pip`
 
-     ```bash
-     pip install element-calcium-imaging
-     ```
+  ```bash
+  pip install -e .
+  ```
 
 + [Interactive tutorial on GitHub Codespaces](#interactive-tutorial)
 

docs/src/index.md

@@ -27,11 +27,19 @@ schemas can be found in the [Data Pipeline](./pipeline.md) documentation page.
 
 ## Getting Started
 
-+ Install from PyPI
++ Please fork the [repository](https://github.com/datajoint/element-calcium-imaging){:target="_blank"}
 
-     ```bash
-     pip install element-calcium-imaging
-     ```
++ Clone the repository to your computer
+
+  ```bash
+  git clone https://github.com/<enter_github_username>/element-calcium-imaging
+  ```
+
++ Install with `pip`
+
+  ```bash
+  pip install -e .
+  ```
 
 + [Data Pipeline](./pipeline.md) - Pipeline and table descriptions
 

element_calcium_imaging/imaging.py

@@ -9,7 +9,7 @@
 
 from . import imaging_report, scan
 from .scan import (
-    get_image_files,
+    get_calcium_imaging_files,
     get_imaging_root_data_dir,
     get_processed_root_data_dir,
 )

element_calcium_imaging/imaging_no_curation.py

@@ -9,7 +9,7 @@
 
 from . import imaging_report, scan
 from .scan import (
-    get_image_files,
+    get_calcium_imaging_files,
     get_imaging_root_data_dir,
     get_processed_root_data_dir,
 )

element_calcium_imaging/imaging_preprocess.py

@@ -9,7 +9,7 @@
 
 from . import imaging_report, scan
 from .scan import (
-    get_image_files,
+    get_calcium_imaging_files,
     get_imaging_root_data_dir,
     get_processed_root_data_dir,
 )

element_calcium_imaging/scan.py

@@ -104,16 +104,17 @@ def get_processed_root_data_dir() -> Union[str, pathlib.Path]:
         return get_imaging_root_data_dir()[0]
 
 
-def get_image_files(scan_key: dict, file_type: str) -> list:
-    """Retrieve the list of image files associated with a given Scan.
+def get_calcium_imaging_files(scan_key: dict, acq_software: str) -> list:
+    """Retrieve the list of absolute paths of the calcium imaging files associated with a given Scan and a given acquisition software (e.g. .tif, .sbx, etc.).
 
     Args:
         scan_key: Primary key of a Scan entry.
+        acq_software: name of the acquisition software, matching AcquisitionSoftware entry (i.e. "ScanImage" "Scanbox", "NIS", or "PrairieView")
 
     Returns:
         A list of full file paths.
     """
-    return _linking_module.get_image_files(scan_key, file_type)
+    return _linking_module.get_calcium_imaging_files(scan_key, acq_software)
 
 
 # ----------------------------- Table declarations ----------------------
@@ -289,12 +290,12 @@ def make(self, key):
         """Populate the ScanInfo with the information parsed from image files."""
 
         acq_software = (Scan & key).fetch1("acq_software")
+        scan_filepaths = get_calcium_imaging_files(key, acq_software)
 
         if acq_software == "ScanImage":
             import scanreader
 
             # Read the scan
-            scan_filepaths = get_image_files(key, "*.tif")
             scan = scanreader.read_scan(scan_filepaths)
 
             # Insert in ScanInfo
@@ -370,7 +371,6 @@ def make(self, key):
             import sbxreader
 
             # Read the scan
-            scan_filepaths = get_image_files(key, "*.sbx")
             sbx_meta = sbxreader.sbx_get_metadata(scan_filepaths[0])
             sbx_matinfo = sbxreader.sbx_get_info(scan_filepaths[0])
             is_multiROI = bool(
@@ -429,7 +429,6 @@ def make(self, key):
             import nd2
 
             # Read the scan
-            scan_filepaths = get_image_files(key, "*.nd2")
             nd2_file = nd2.ND2File(scan_filepaths[0])
             is_multiROI = False  # MultiROI to be implemented later
 
@@ -519,7 +518,6 @@ def estimate_nd2_scan_duration(nd2_scan_obj):
         elif acq_software == "PrairieView":
             from element_interface import prairie_view_loader
 
-            scan_filepaths = get_image_files(key, "*.tif")
             PVScan_info = prairie_view_loader.get_prairieview_metadata(
                 scan_filepaths[0]
             )
@@ -612,30 +610,31 @@ def make(self, key):
         acq_software, nchannels = (Scan * ScanInfo & key).fetch1(
             "acq_software", "nchannels"
         )
+        scan_filepaths = get_calcium_imaging_files(key, acq_software)
 
         if acq_software == "ScanImage":
             import scanreader
 
             # Switch from FYXCT to TCYX
-            data = scanreader.read_scan(get_image_files(key, "*.tif"))[
-                key["field_idx"]
-            ].transpose(3, 2, 0, 1)
+            data = scanreader.read_scan(scan_filepaths)[key["field_idx"]].transpose(
+                3, 2, 0, 1
+            )
         elif acq_software == "Scanbox":
             from sbxreader import sbx_memmap
 
             # Switch from TFCYX to TCYX
-            data = sbx_memmap(get_image_files(key, "*.sbx"))[:, key["field_idx"]]
+            data = sbx_memmap(scan_filepaths)[:, key["field_idx"]]
         elif acq_software == "NIS":
             import nd2
 
-            nd2_file = nd2.ND2File(get_image_files(key, "*.nd2")[0])
+            nd2_file = nd2.ND2File(scan_filepaths[0])
 
             nd2_dims = {k: i for i, k in enumerate(nd2_file.sizes)}
 
             valid_dimensions = "TZCYX"
             assert set(nd2_dims) <= set(
                 valid_dimensions
-            ), f"Unknown dimensions {set(nd2_dims)-set(valid_dimensions)} in file {get_image_files(key, '*.nd2')[0]}."
+            ), f"Unknown dimensions {set(nd2_dims)-set(valid_dimensions)} in file {scan_filepaths[0]}."
 
             # Sort the dimensions in the order of TZCYX, skipping the missing ones.
             data = nd2_file.asarray().transpose(

element_calcium_imaging/version.py

@@ -1,2 +1,2 @@
 """Package metadata."""
-__version__ = "0.7.9"
+__version__ = "0.8.0"

notebooks/tutorial_pipeline.py

@@ -26,18 +26,27 @@ def get_imaging_root_data_dir():
         raise TypeError("`imaging_root_data_dir` must be a string, pathlib, or list")
 
 
-def get_image_files(scan_key, file_type: str):
-    """Retrieve the list of absolute paths associated with a given Scan."""
+def get_calcium_imaging_files(scan_key, acq_software: str):
+    """Retrieve the list of absolute paths of the calcium imaging files associated with a given Scan and a given acquisition software (e.g. "ScanImage", "PrairieView", etc.)."""
     # Folder structure: root / subject / session / .tif or .sbx or .nd2
     session_dir = element_interface.utils.find_full_path(
         get_imaging_root_data_dir(),
         (session.SessionDirectory & scan_key).fetch1("session_dir"),
     )
 
-    filepaths = [fp.as_posix() for fp in session_dir.glob(file_type)]
+    if acq_software == "ScanImage":
+        filepaths = [fp.as_posix() for fp in session_dir.glob("*.tif")]
+    elif acq_software == "Scanbox":
+        filepaths = [fp.as_posix() for fp in session_dir.glob("*.sbx")]
+    elif acq_software == "NIS":
+        filepaths = [fp.as_posix() for fp in session_dir.glob("*.nd2")]
+    elif acq_software == "PrairieView":
+        filepaths = [fp.as_posix() for fp in session_dir.glob("*.tif")]
+    else:
+        raise NotImplementedError(f"{acq_software} is not implemented")
 
     if not filepaths:
-        raise FileNotFoundError(f"No {file_type} file found in {session_dir}")
+        raise FileNotFoundError(f"No {acq_software} file found in {session_dir}")
     return filepaths