Get it working again for newer versions of dependencies

.github/workflows/github-actions-ci.yml

@@ -20,21 +20,22 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14-dev"]
 
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
+      - name: Install system packages
         run: |
           sudo apt-get update
-          sudo apt-get install -y hdf5-tools curl
+          sudo apt-get install -y hdf5-tools curl libhdf5-dev
+      - name: Install pip dependencies
+        run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
-          pip install allensdk
       - name: Run tests
         run: |
           pip install -r requirements-test.txt
@@ -51,7 +52,7 @@ jobs:
     runs-on: ["self-hosted"]
     strategy:
       matrix:
-        pyver: ["3.9" , "3.11"]
+        pyver: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14-rc"]
     steps:
       - uses: actions/checkout@v4
       - name: Build docker image

.github/workflows/nightly-onprem.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ["self-hosted"]
     strategy:
       matrix:
-        pyver: ["3.9" , "3.11"]
+        pyver: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14-rc"]
         branch: ["master", "feature/**"]
     steps:
       - uses: actions/checkout@v4

docker/Dockerfile

@@ -9,6 +9,7 @@ RUN apt-get update                    \
     && apt-get install -y             \
         hdf5-tools                    \
         curl                          \
+        libhdf5-dev                   \
         git-lfs                       \
         && rm -rf /var/lib/apt/lists/*
 

ipfx/attach_metadata/sink/nwb2_sink.py

@@ -115,8 +115,8 @@ def _get_single_ic_electrode(self) -> pynwb.icephys.IntracellularElectrode:
 
         """
 
-        keys = list(self.nwbfile.ic_electrodes.keys())
-        
+        keys = list(self.nwbfile.icephys_electrodes.keys())
+
         if len(keys) != 1:
             raise ValueError(
                 "expected exactly 1 intracellular electrode, found "

ipfx/bin/generate_fx_input.py

@@ -1,5 +1,5 @@
 import os
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 import ipfx.sweep_props as sp
 from ipfx.bin.run_sweep_extraction import run_sweep_extraction
 from ipfx.bin.generate_qc_input import generate_qc_input

ipfx/bin/generate_pipeline_input.py

@@ -1,5 +1,5 @@
 import ipfx.qc_feature_evaluator as qcp
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 import os.path
 from ipfx.bin.generate_se_input import generate_se_input, parse_args
 import ipfx.lims_queries as lq

ipfx/bin/generate_qc_input.py

@@ -2,7 +2,7 @@
 from ipfx.bin.run_sweep_extraction import run_sweep_extraction
 from ipfx.bin.generate_se_input import generate_se_input, parse_args
 import ipfx.sweep_props as sp
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 import ipfx.logging_utils as lu
 
 

ipfx/bin/generate_se_input.py

@@ -1,4 +1,4 @@
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 import os
 import ipfx.lims_queries as lq
 import argparse

ipfx/bin/make_stimulus_ontology.py

@@ -1,4 +1,4 @@
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 from ipfx.stimulus import StimulusOntology
 import re
 import ipfx.lims_queries as lq
@@ -84,5 +84,5 @@ def main():
     make_default_stimulus_ontology()
 
 
-if __name__== "__main__": 
+if __name__== "__main__":
     main()

ipfx/bin/pipeline_from_specimen_id.py

@@ -1,4 +1,4 @@
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 import sys
 import os.path
 from .run_pipeline import run_pipeline
@@ -34,19 +34,21 @@ def main():
     pipe_input = gpi.generate_pipeline_input(cell_dir,
                                              specimen_id=int(specimen_id))
 
-    input_json = os.path.join(cell_dir,INPUT_JSON)
-    ju.write(input_json,pipe_input)
+    input_json = os.path.join(cell_dir, INPUT_JSON)
+    ju.write(input_json, pipe_input)
 
     #   reading back from disk
     pipe_input = ju.read(input_json)
-    pipe_output = run_pipeline(pipe_input["input_nwb_file"],
-                          pipe_input["output_nwb_file"],
-                          pipe_input.get("stimulus_ontology_file", None),
-                          pipe_input.get("qc_fig_dir",None),
-                          pipe_input["qc_criteria"],
-                          pipe_input["manual_sweep_states"])
-
-    ju.write(os.path.join(cell_dir,OUTPUT_JSON), pipe_output)
+    pipe_output = run_pipeline(
+        pipe_input["input_nwb_file"],
+        pipe_input["output_nwb_file"],
+        pipe_input.get("stimulus_ontology_file", None),
+        pipe_input.get("qc_fig_dir",None),
+        pipe_input["qc_criteria"],
+        pipe_input["manual_sweep_states"],
+    )
+
+    ju.write(os.path.join(cell_dir, OUTPUT_JSON), pipe_output)
 
 if __name__ == "__main__": main()
 

ipfx/bin/plot_ephys_nwb.py

@@ -6,7 +6,7 @@
 from ipfx.qc_feature_extractor import sweep_qc_features
 from ipfx.utilities import drop_failed_sweeps
 from ipfx.stimulus import StimulusOntology
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 from typing import (
     Optional, List, Dict, Tuple, Collection, Sequence, Union
 )
@@ -27,11 +27,11 @@ def plot_data_set(data_set,
         data_set.sweep_info = sweep_qc_features(data_set)
 
     sweep_table = data_set.filtered_sweep_table(clamp_mode=clamp_mode, stimuli=stimuli, stimuli_exclude=stimuli_exclude)
-    
+
     if len(sweep_table)==0:
         warnings.warn("No sweeps to plot")
         return
-    
+
     height_ratios, width_ratios = axes_ratios(sweep_table)
 
     fig, ax = plt.subplots(len(height_ratios), 3,
@@ -51,7 +51,7 @@ def plot_data_set(data_set,
         annot = sweep_numbers.astype(str)
         if show_amps:
             annot += sweep_set_table['stimulus_amplitude'].apply(": {:.3g} pA".format)
-            
+
 
         ax_a = ax[fig_row,0]
         ax_i = ax[fig_row,1]

ipfx/bin/run_chirp_fv_extraction.py

@@ -4,7 +4,7 @@
 import traceback
 from multiprocessing import Pool
 from functools import partial
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 from ipfx.stimulus import StimulusOntology
 import ipfx.script_utils as su
 
@@ -32,9 +32,9 @@ class CollectChirpFeatureVectorParameters(ags.ArgSchema):
         ],
         cli_as_single_argument=True)
     data_source = ags.fields.String(
-        description="Source of NWB files ('sdk' or 'lims')",
-        default="sdk",
-        validate=lambda x: x in ["sdk", "lims"]
+        description="Source of NWB files ('lims' is only currently implemented option)",
+        default="lims",
+        validate=lambda x: x in ["lims"]
         )
 
 
@@ -142,4 +142,4 @@ def main(output_dir, output_code, input_file, include_failed_cells,
 
 if __name__ == "__main__":
     module = ags.ArgSchemaParser(schema_type=CollectChirpFeatureVectorParameters)
-    main(**module.args)
+    main(**module.args)

ipfx/bin/run_feature_collection.py

@@ -6,7 +6,7 @@
 import ipfx.feature_vectors as fv
 import ipfx.script_utils as su
 from ipfx.stimulus import StimulusOntology
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 import logging
 from multiprocessing import Pool
 from functools import partial
@@ -20,9 +20,9 @@ class CollectFeatureParameters(ags.ArgSchema):
     include_failed_cells = ags.fields.Boolean(default=False)
     run_parallel = ags.fields.Boolean(default=True)
     data_source = ags.fields.String(
-        description="Source of NWB files ('sdk' or 'lims' or 'filesystem')",
-        default="sdk",
-        validate=lambda x: x in ["sdk", "lims", "filesystem"]
+        description="Source of NWB files ('lims' or 'filesystem')",
+        default="lims",
+        validate=lambda x: x in ["lims", "filesystem"]
         )
 
 
@@ -97,7 +97,7 @@ def extract_features(data_set, ramp_sweep_numbers, ssq_sweep_numbers, lsq_sweep_
         mask_supra = sweep_table["stim_amp"] >= basic_lsq_features["rheobase_i"]
         sweep_indexes = fv._consolidated_long_square_indexes(sweep_table.loc[mask_supra, :])
         amps = np.rint(sweep_table.loc[sweep_indexes, "stim_amp"].values - basic_lsq_features["rheobase_i"])
-        spike_data = np.array(basic_lsq_features["spikes_set"])
+        spike_data = np.array(basic_lsq_features["spikes_set"], dtype=object)
 
         for amp, swp_ind in zip(amps, sweep_indexes):
             if (amp % amp_interval != 0) or (amp > max_above_rheo) or (amp < 0):

ipfx/bin/run_feature_extraction.py

@@ -7,7 +7,7 @@
 from ipfx._schemas import FeatureExtractionParameters
 from ipfx.dataset.create import create_ephys_data_set
 import ipfx.sweep_props as sp
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 from ipfx.nwb_append import append_spike_times
 
 import ipfx.plot_qc_figures as plotqc

ipfx/bin/run_feature_vector_extraction.py

@@ -7,7 +7,7 @@
 import os
 import h5py
 from ipfx.stimulus import StimulusOntology
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 import ipfx.feature_vectors as fv
 import ipfx.lims_queries as lq
 import ipfx.script_utils as su
@@ -26,9 +26,9 @@ class CollectFeatureVectorParameters(ags.ArgSchema):
         allow_none=True
     )
     data_source = ags.fields.String(
-        description="Source of NWB files ('sdk' or 'lims' or 'filesystem')",
-        default="sdk",
-        validate=lambda x: x in ["sdk", "lims", "filesystem"]
+        description="Source of NWB files ('lims' or 'filesystem')",
+        default="lims",
+        validate=lambda x: x in ["lims", "filesystem"]
         )
     output_code = ags.fields.String(
         description="Code used for naming of output files",

ipfx/bin/run_pipeline.py

@@ -1,8 +1,7 @@
 import logging
 import argschema as ags
 
-import allensdk.core.json_utilities as json_utilities
-
+import ipfx.json_utilities as json_utilities
 import ipfx.sweep_props as sweep_props
 from ipfx.logging_utils import log_pretty_header
 from ipfx._schemas import PipelineParameters

ipfx/bin/run_pipeline_from_nwb_file.py

@@ -1,4 +1,4 @@
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 import os.path
 from ipfx.bin.run_pipeline import run_pipeline
 from ipfx.bin.generate_pipeline_input import generate_pipeline_input
@@ -37,7 +37,7 @@ def main():
         )
     )
     parser.add_argument(
-        "--output_json", type=str, default="output.json", 
+        "--output_json", type=str, default="output.json",
         help=(
             "write output json file here (relative to OUTPUT_DIR/cell_name, "
             "where cell_name is the extensionless basename of the input NWB "
@@ -49,7 +49,7 @@ def main():
         help=(
             "Generate qc figures and store them here (relative to "
             "OUTPUT_DIR/cell_name, where cell_name is the extensionless "
-            "basename of the input nwb file). If you supply --qc_fig_dir with " 
+            "basename of the input nwb file). If you supply --qc_fig_dir with "
             "no arguments, the path will be OUTPUT_DIR/cell_name/qc_figs. If "
             "this argument is not supplied, no figures will be generated."
         )

ipfx/bin/run_qc.py

@@ -4,7 +4,7 @@
 import ipfx.qc_feature_evaluator as qcp
 import argschema as ags
 from ipfx._schemas import QcParameters
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 import ipfx.sweep_props as sp
 import pandas as pd
 import ipfx.logging_utils as lu

ipfx/bin/run_sweep_extraction.py

@@ -1,7 +1,7 @@
 import logging
 
-import allensdk.core.json_utilities as json_utilities
 import argschema as ags
+import ipfx.json_utilities as json_utilities
 
 from ipfx._schemas import SweepExtractionParameters
 from ipfx.dataset.create import create_ephys_data_set

ipfx/bin/validate_experiment.py

@@ -1,8 +1,8 @@
-import allensdk.core.json_utilities as ju
-import numpy as np
 import sys
 import os
 import logging
+import numpy as np
+import ipfx.json_utilities as ju
 
 
 def nullisclose(a, b):

ipfx/data_set_utils.py

@@ -1,7 +1,7 @@
 """A shim for backwards compatible imports of create_data_set
 """
 
-from allensdk.deprecated import deprecated
+from ipfx.deprecated import deprecated
 
 from ipfx.dataset.create import create_ephys_data_set
 create_data_set = deprecated(  # type: ignore

ipfx/dataset/create.py

@@ -5,7 +5,7 @@
 import h5py
 import numpy as np
 
-import allensdk.core.json_utilities as ju
+import ipfx.json_utilities as ju
 
 from ipfx.dataset.ephys_data_set import EphysDataSet
 from ipfx.stimulus import StimulusOntology
@@ -17,7 +17,7 @@
 
 def get_scalar_value(dataset_from_nwb):
     """
-    Some values in NWB are stored as scalar whereas others as np.ndarrays with 
+    Some values in NWB are stored as scalar whereas others as np.ndarrays with
     dimension 1. Use this function to retrieve the scalar value itself.
     """
 
@@ -67,7 +67,7 @@ def get_nwb_version(nwb_file: str) -> Dict[str, Any]:
             nwb_version_str = to_str(nwb_version)
             if nwb_version is not None and re.match("^NWB-", nwb_version_str):
                 return {
-                    "major": int(nwb_version_str[4]), 
+                    "major": int(nwb_version_str[4]),
                     "full": nwb_version_str
                 }