Merge branch 'main' of https://github.com/AllenInstitute/openscope_databook

Author: rcpeene

.github/workflows/build.yml

@@ -4,14 +4,16 @@ on:
   push:
     branches: [ "main" ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main" , "dev"]
 
   workflow_dispatch:
 
 jobs:
   build:
     runs-on:
       group: LargerInstance
+    container: 
+      image: rcpeene/openscope_databook:latest
 
     env:
       DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }}
@@ -21,21 +23,27 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-          ref: main
+          ref: ${{ github.ref }}
 
       # - name: Set up Python
       #   uses: actions/setup-python@v4
       #   with:
-      #     python-version: "3.10"
+      #     python-version: "3.11"
+
+      # - name: Upgrading pip
+      #   run: pip install --upgrade pip
 
-      - name: Upgrading pip
-        run: pip install --upgrade pip
+      # - name: Install deps
+      #   run: pip install cython numpy
 
-      - name: Install deps
-        run: pip install cython numpy
+      - name: pip freeze
+        run: pip freeze
 
-      - name: Installing package
-        run: pip install -e .
+      # - name: Installing packages again (this prevents a weird error)
+      #   run: pip install -r requirements.txt
+
+      # - name: Installing package
+      #   run: pip install -e .
 
       - name: Installing build dependencies
         run: |
@@ -79,7 +87,9 @@ jobs:
           rm ./docs/embargoed/*.nwb        
 
       - name: Printing log
-        run: git status
+        run: |
+          git config --global --add safe.directory /__w/openscope_databook/openscope_databook
+          git status
       
       - name: Printing shortlog
         run: git log | git shortlog -sn  

.github/workflows/deploy.yml

@@ -12,12 +12,20 @@ jobs:
         with:
           fetch-depth: 0
           ref: main
-      
+
+      - name: Installing python
+        run: |
+          sudo apt-get update
+          sudo apt install python3.12
+          sudo apt-get install build-essential
+
       - name: Upgrading pip
         run: pip install --upgrade pip
 
       - name: Installing dependencies
-        run: pip install markupsafe==2.0.1 numpy cython
+        run: |
+          pip install --upgrade pip setuptools
+          pip install markupsafe==2.0.1 numpy cython --no-cache-dir
 
       - name: Installing package
         run: pip install -e .
@@ -36,7 +44,7 @@ jobs:
           jupyter-book clean ./docs
           jupyter-book build ./docs
             
-      - name: Deploy book to Github pages
+      - name: Deploy book to GitHub pages
         uses: peaceiris/[email protected]
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -11,6 +11,8 @@ jobs:
   test:
     runs-on: 
       group: LargerInstance
+    container:
+      image: rcpeene/openscope_databook:latest
 
     env:
       DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }}
@@ -19,20 +21,20 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Upgrading pip
-        run: pip install --upgrade pip
+      # - name: Upgrading pip
+      #   run: pip install --upgrade pip
 
       - name: print environment
         run: pip freeze
 
-      - name: Install cython
-        run: pip install cython numpy
+      # - name: Install cython
+      #   run: pip install cython numpy
 
-      - name: Installing package
-        run: pip install -e .
+      # - name: Installing package
+      #   run: pip install -e .
 
-      - name: Installing requirements
-        run: pip install -r ./requirements.txt
+      # - name: Installing requirements
+      #   run: pip install -r ./requirements.txt
 
       - name: Installing build dependencies
         run: |

Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:22.04
+# base requirements
+RUN apt-get update
+RUN apt-get install -y coreutils
+RUN apt-get install -y libgl1-mesa-glx  
+RUN apt-get install -y libglib2.0-0
+RUN apt-get install -y python3 python3-pip
+RUN apt-get install -y git
+
+RUN git config --global --add safe.directory /__w/openscope_databook/openscope_databook
+
+# copy databook setup files
+COPY requirements.txt ./openscope_databook/requirements.txt
+COPY setup.py ./openscope_databook/setup.py
+COPY README.md ./openscope_databook/README.md
+COPY LICENSE.txt ./openscope_databook/LICENSE.txt
+COPY databook_utils ./openscope_databook/databook_utils
+
+# for reasons I don't understand, these must be installed before the rest the requirements
+RUN pip install numpy cython
+# set up databook dependencies
+RUN pip install -e ./openscope_databook[dev]

README.md

@@ -2,7 +2,9 @@
 
 ### **The Deployed Databook can be found here: https://alleninstitute.github.io/openscope_databook**
 
-The OpenScope Databook is meant to store code and documentation used for reproducible brain data analysis and visualization, primarily working with [NWB files](https://www.nwb.org/how-to-use/) and the [DANDI archive](https://dandiarchive.org/). It is provided by the Allen Institute's **[OpenScope](https://alleninstitute.org/what-we-do/brain-science/research/mindscope-program/openscope/)** Project, a component of The [Allen Institute for Neuraldynamics](https://alleninstitute.org/division/neural-dynamics/). **OpenScope** is a platform for high-throughput and reproducible neurophysiology open to external scientists to test theories of brain function. Through [Jupyter Book](https://jupyterbook.org/), this code is structured as a series of documented Jupyter notebooks intended to explain and educate users on how to work with brain data.
+### **The citeable DOI page on Zenodo can be found here: https://zenodo.org/records/12614664**
+
+The OpenScope Databook is meant to store code and documentation used for reproducible brain data analysis and visualization, primarily working with [NWB files](https://www.nwb.org/how-to-use/) and the [DANDI archive](https://dandiarchive.org/). It is provided by the Allen Institute's **[OpenScope](https://alleninstitute.org/what-we-do/brain-science/research/mindscope-program/openscope/)** Project, a component of The [Allen Institute for Neural Dynamics](https://alleninstitute.org/division/neural-dynamics/). **OpenScope** is a platform for high-throughput and reproducible neurophysiology open to external scientists to test theories of brain function. Through [Jupyter Book](https://jupyterbook.org/), this code is structured as a series of documented Jupyter notebooks intended to explain and educate users on how to work with brain data.
 
 We are releasing this code to the public as a tool we expect others to use and are actively updating and maintaining this project. Issue submissions are encouraged. Questions can be directed to [@rcpeene](https://github.com/rcpeene) or [@jeromelecoq](https://github.com/jeromelecoq). Below, you can see a working list of the content goals for this databook. We are open to hearing input from users about what types of analysis and visualization might be useful for reproducible neuroscience, particularly when working with the *NWB* file standard.
 
@@ -18,7 +20,7 @@ We are releasing this code to the public as a tool we expect others to use and a
 ### Chapter 2: Data visualization
 - [Visualizing eye-tracking gaze locations, eye area, and running speed](https://alleninstitute.github.io/openscope_databook/embargoed/visualize_behavior.html)
 - [Visualizing 2P raw movie](https://alleninstitute.github.io/openscope_databook/visualization/visualize_2p_raw.html)
-- [Visualizing Neuropixels probe locations](https://alleninstitute.github.io/openscope_databook/visualization/visualize_neuropixel_probes.html)
+- [Visualizing Neuropixels probe locations](https://alleninstitute.github.io/openscope_databook/visualization/visualize_neuropixels_probes.html)
 - [Visualizing Neuropixels recorded unit quality metrics](https://alleninstitute.github.io/openscope_databook/visualization/visualize_unit_metrics.html)
 - [Visualizing LFP responses to stimulus events](https://alleninstitute.github.io/openscope_databook/visualization/visualize_lfp_responses.html)
 - [Visualizing neuronal spike matrices](https://alleninstitute.github.io/openscope_databook/visualization/visualize_unit_spikes.html)
@@ -31,19 +33,19 @@ We are releasing this code to the public as a tool we expect others to use and a
 - [Neuropixels: Extracting Current Source Density plots](https://alleninstitute.github.io/openscope_databook/first-order/current_source_density.html)
 - [2P: Cell matching across days](https://alleninstitute.github.io/openscope_databook/embargoed/cell_matching.html)
 - [2P: Stimuli averages with 2P data](https://alleninstitute.github.io/openscope_databook/first-order/test_2p_responses.html)
-- [Neuropixels: Stimuli averages with neuropixels data](https://alleninstitute.github.io/openscope_databook/first-order/test_spike_responses.html)
+- [Neuropixels: Stimuli averages with neuropixels data](https://alleninstitute.github.io/openscope_databook/first-order/test_unit_responses.html)
 - [2P: How to align timestamps across modalities](https://alleninstitute.github.io/openscope_databook/embargoed/modality_alignment.html)
 - [Sending NWB raw data to a segmentation pipeline: example with Suite2p](https://alleninstitute.github.io/openscope_databook/first-order/suite2p.html)
-- [Classifying spike waveform between fast spiking and normal spiking cells](https://github.com/AllenInstitute/openscope_databook/blob/main/docs/first-order/classify_waveforms.ipynb)
+- [Classifying spike waveform between fast spiking and normal spiking cells](https://alleninstitute.github.io/openscope_databook/first-order/classify_waveforms.html)
 
 ### Chapter 4: Higher-order analysis
 - [Neural dynamics using time-analysis with CEBRA](https://alleninstitute.github.io/openscope_databook/higher-order/cebra_time.html)  
   [CEBRA Demo on the CEBRA Repository](https://github.com/adaptivemotorcontrollab/CEBRA-demos/blob/main/Demo_openscope_databook.ipynb)
 - [Extracting clusters of correlated neurons with TCA](https://alleninstitute.github.io/openscope_databook/higher-order/tca.html)
 - [Estimating behavioral state given trial choices and relating states to pupil size](https://alleninstitute.github.io/openscope_databook/higher-order/behavioral_state.html)
-- Analysis of functional connectivity.
 
 ### Chapter 5: Replicating figures
-- [Openscope Credit Assignment](https://alleninstitute.github.io/openscope_databook/replication/cred_assign_figures.html)
-- Example notebooks from past projects
-- Guidelines for reproducible figures from NWB files
+- [OpenScope Credit Assignment](https://alleninstitute.github.io/openscope_databook/projects/cred_assign_figures.html)
+- [OpenScope Global/Local Oddball](https://alleninstitute.github.io/openscope_databook/projects/glo.html)
+- [OpenScope Illusion](https://alleninstitute.github.io/openscope_databook/projects/illusion.html)
+- [OpenScope Dendritic Coupling](https://alleninstitute.github.io/openscope_databook/projects/dendritic_coupling.html)