Convert behavior only recordings (#134)

* behavior_only option

* docstring and notebook update

* add test and better error message

* update data cache key to force new download

* suggestions from review

* Apply suggestion from @Copilot

Co-authored-by: Copilot <[email protected]>

---------

Co-authored-by: Copilot <[email protected]>

Author: samuelbray32

.github/workflows/test_package_build.yml

@@ -104,8 +104,8 @@ jobs:
         with:
           # Path to cache
           path: ${{ env.DOWNLOAD_DIR }}
-          # Cache key: OS + static string + version number (bump v1 to v2 to invalidate)
-          key: ${{ runner.os }}-testdata-trodes-v1
+          # Cache key: OS + static string + version number (bump v2 to v3 to invalidate)
+          key: ${{ runner.os }}-testdata-trodes-v2
 
       # --- Download Test Data Step (Conditional) ---
       - name: Download test rec files

notebooks/conversion_tutorial.ipynb

@@ -48,6 +48,10 @@
                 "Where the nwb file will be saved. For Frank Lab members, this should be `/stelmo/nwb/raw/`. File is named by the convention {animal}{date}.nwb\n",
                 "\n",
                 "+ `query_expression`: A query expression to select which files to convert. For example, if you have several animals in your folder, you could write `\"animal == 'sample'\"` to select only the sample animal. Defaults to `None` which converts all files in the directory (potentially overwriting ones you've done before!).\n",
+                "\n",
+                "+ `behavior_only`: Rec files recorded through trodes software without e-phys data have a\n",
+                "   different expected internal structure. Use this flag to ensure correct data parsing \n",
+                "   is used. Default of `False` expects electrophysiology data to be present\n",
                 "\n"
             ]
         },

src/trodes_to_nwb/convert.py

@@ -112,6 +112,7 @@ def create_nwbs(
     n_workers: int = 1,
     query_expression: str | None = None,
     disable_ptp: bool = False,
+    behavior_only: bool = False,
 ):
     """
     Convert SpikeGadgets data to NWB format.
@@ -138,6 +139,9 @@ def create_nwbs(
         See https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.query.html.
     disable_ptp : bool, optional
         Blocks use of ptp timestamps regardless of rec header, by default False.
+    behavior_only : bool, optional
+        Flag to indicate only behaviorsl data (no ephys) was collected in the rec
+        files, by default False.
 
     """
 
@@ -166,6 +170,7 @@ def pass_func(args):
                     output_dir,
                     video_directory,
                     convert_video,
+                    behavior_only=behavior_only,
                 )
                 return True
             except Exception as e:
@@ -194,6 +199,7 @@ def pass_func(args):
                 video_directory,
                 convert_video,
                 disable_ptp,
+                behavior_only=behavior_only,
             )
 
 
@@ -206,6 +212,7 @@ def _create_nwb(
     video_directory: str = "",
     convert_video: bool = False,
     disable_ptp: bool = False,
+    behavior_only: bool = False,
 ):
     # create loggers
     logger = setup_logger("convert", f"{session[1]}{session[0]}_convert.log")
@@ -217,7 +224,10 @@ def _create_nwb(
     logger.info("CREATING REC DATA ITERATORS")
     # make generic rec file data chunk iterator to pass to functions
     rec_dci = RecFileDataChunkIterator(
-        rec_filepaths, interpolate_dropped_packets=False, stream_id="trodes"
+        rec_filepaths,
+        interpolate_dropped_packets=False,
+        stream_id="ECU_analog" if behavior_only else "trodes",
+        behavior_only=behavior_only,
     )
     rec_dci_timestamps = (
         rec_dci.timestamps
@@ -263,30 +273,36 @@ def _create_nwb(
     add_acquisition_devices(nwb_file, metadata)
     add_tasks(nwb_file, metadata)
     add_associated_files(nwb_file, metadata)
-    add_electrode_groups(
-        nwb_file, metadata, device_metadata, hw_channel_map, ref_electrode_map
-    )
     add_header_device(nwb_file, rec_header)
     add_associated_video_files(
         nwb_file, metadata, session_df, video_directory, convert_video
     )
     add_optogenetics(nwb_file, metadata, device_metadata)
 
-    logger.info("ADDING EPHYS DATA")
-    # add rec file data
-    map_row_ephys_data_to_row_electrodes_table = list(
-        range(len(nwb_file.electrodes))
-    )  # TODO: Double check this
-    add_raw_ephys(
-        nwb_file,
-        rec_filepaths,
-        map_row_ephys_data_to_row_electrodes_table,
-        metadata,
-    )
+    if not behavior_only:
+        add_electrode_groups(
+            nwb_file, metadata, device_metadata, hw_channel_map, ref_electrode_map
+        )
+        logger.info("ADDING EPHYS DATA")
+        # add rec file data
+        map_row_ephys_data_to_row_electrodes_table = list(
+            range(len(nwb_file.electrodes))
+        )  # TODO: Double check this
+        add_raw_ephys(
+            nwb_file,
+            rec_filepaths,
+            map_row_ephys_data_to_row_electrodes_table,
+            metadata,
+        )
     logger.info("ADDING DIO DATA")
     add_dios(nwb_file, rec_filepaths, metadata)
     logger.info("ADDING ANALOG DATA")
-    add_analog_data(nwb_file, rec_filepaths, timestamps=rec_dci_timestamps)
+    add_analog_data(
+        nwb_file,
+        rec_filepaths,
+        timestamps=rec_dci_timestamps,
+        behavior_only=behavior_only,
+    )
     logger.info("ADDING SAMPLE COUNTS")
     add_sample_count(nwb_file, rec_dci)
     logger.info("ADDING EPOCHS")

src/trodes_to_nwb/convert_analog.py

@@ -15,7 +15,11 @@
 
 
 def add_analog_data(
-    nwbfile: NWBFile, rec_file_path: list[str], timestamps: np.ndarray = None, **kwargs
+    nwbfile: NWBFile,
+    rec_file_path: list[str],
+    timestamps: np.ndarray = None,
+    behavior_only: bool = False,
+    **kwargs,
 ) -> None:
     """Adds analog streams to the nwb file.
 
@@ -46,9 +50,10 @@ def add_analog_data(
     rec_dci = RecFileDataChunkIterator(
         rec_file_path,
         nwb_hw_channel_order=analog_channel_ids,
-        stream_index=2,
+        stream_id="ECU_analog",
         is_analog=True,
         timestamps=timestamps,
+        behavior_only=behavior_only,
     )
 
     # add headstage channel IDs to the list of analog channel IDs

src/trodes_to_nwb/convert_ephys.py

@@ -47,6 +47,7 @@ def __init__(
         is_analog: bool = False,
         interpolate_dropped_packets: bool = False,
         timestamps=None,  # Use this if you already have timestamps from intializing another rec iterator on the same files
+        behavior_only: bool = False,
         **kwargs,
     ):
         """
@@ -69,6 +70,8 @@ def __init__(
             whether to interpolate single dropped packets, by default False
         timestamps : [type], optional
             timestamps to use. Can provide efficiency improvements by skipping recalculating timestamps from rec files, by default None
+        behavior_only : bool, optional
+            indicate if file contains only behavior data (no e-phys), by default False
         kwargs : dict
             additional arguments to pass to GenericDataChunkIterator
         """
@@ -94,7 +97,15 @@ def __init__(
         # trodes
         assert all([neo_io.block_count() == 1 for neo_io in self.neo_io])
         assert all([neo_io.segment_count(0) == 1 for neo_io in self.neo_io])
-        assert all([neo_io.signal_streams_count() == 4 for neo_io in self.neo_io])
+        assert all(
+            [
+                neo_io.signal_streams_count() == 4 - behavior_only
+                for neo_io in self.neo_io
+            ]
+        ), (
+            "Unexpected number of signal streams. "
+            + "Confirm whether behavior_only is set correctly for this recording"
+        )
 
         self.block_index = 0
         self.seg_index = 0
@@ -113,6 +124,10 @@ def __init__(
         else:  # if stream id is not provided
             stream_id = self.neo_io[0].get_stream_id_from_index(stream_index)
 
+        if behavior_only and stream_id == "trodes":
+            raise ValueError(
+                "Behavior only recordings do not contain a `trodes` stream"
+            )
         self.stream_id = stream_id
         self.stream_index = stream_index
 

src/trodes_to_nwb/convert_optogenetics.py

@@ -32,16 +32,15 @@ def add_optogenetics(nwbfile: NWBFile, metadata: dict, device_metadata: List[dic
         List of dictionaries containing metadata for devices used in the experiment.
     """
     logger = logging.getLogger("convert")
-    if not all(
-        [
-            x in metadata
-            for x in [
-                "virus_injection",
-                "opto_excitation_source",
-                "optical_fiber",
-                "optogenetic_stimulation_software",
-            ]
-        ]
+    necessary_metadata = [
+        "virus_injection",
+        "opto_excitation_source",
+        "optical_fiber",
+        "optogenetic_stimulation_software",
+    ]
+
+    if not (
+        all([((x in metadata) and len(metadata[x]) > 0) for x in necessary_metadata])
     ):
         logger.info("No available optogenetic metadata")
         return

src/trodes_to_nwb/convert_position.py

@@ -1250,7 +1250,12 @@ def add_associated_video_files(
         epoch = video_metadata["task_epochs"][0]
         # get the video file path
         video_path = None
-        for file in session_df[session_df.file_extension == ".h264"].full_path:
+        for file in session_df[
+            np.logical_or(
+                session_df.file_extension == ".h264",
+                session_df.file_extension == ".mp4",
+            )
+        ].full_path:
             if video_metadata["name"].rsplit(".", 1)[0] in file:
                 video_path = file
                 break

src/trodes_to_nwb/data_scanner.py

@@ -12,6 +12,7 @@
     "rec",  # binary file containing the ephys recording, accelerometer, gyroscope, magnetometer, DIO data, header
     "videoPositionTracking",  # trodes tracked position
     "h264",  # video file
+    "mp4",  # video file
     "cameraHWSync",  # position timestamps
     "stateScriptLog",  # state script controls the experimenter parameters
     "yml",  # metadata file

src/trodes_to_nwb/tests/test_behavior_only_rec.py

@@ -0,0 +1,63 @@
+import pytest
+
+from trodes_to_nwb.convert_ephys import RecFileDataChunkIterator
+from trodes_to_nwb.tests.utils import data_path
+
+
+def test_behavior_only_rec_file():
+    file = data_path / "behavior_only.rec"
+    # accessing trodes stream with behavior only file should fail
+    with pytest.raises(IndexError):
+        RecFileDataChunkIterator(
+            rec_file_path=[file],
+            interpolate_dropped_packets=True,
+            stream_id="trodes",
+            behavior_only=True,
+        )
+
+    # misspecification of behavior-only should result in mismatched stream number
+    with pytest.raises(AssertionError):
+        RecFileDataChunkIterator(
+            rec_file_path=[file],
+            interpolate_dropped_packets=True,
+            stream_id="trodes",
+            behavior_only=False,
+        )
+
+    # correctly build iterator
+    rec_dci = RecFileDataChunkIterator(
+        rec_file_path=[file],
+        interpolate_dropped_packets=True,
+        stream_id="ECU_analog",
+        behavior_only=True,
+    )
+    neo_io = rec_dci.neo_io[0]
+
+    # check file streams
+    stream_names = [stream[0] for stream in neo_io.header["signal_streams"]]
+    assert all(
+        [
+            x in stream_names
+            for x in ["ECU_analog", "ECU_digital", "Controller_DIO_digital"]
+        ]
+    ), "missing expected stream in iterator"
+    assert "trodes" not in stream_names, "unexpected trodes stream in iterator"
+
+    # check data accesses
+    assert rec_dci.timestamps.size == 433012
+    assert rec_dci.timestamps[-1] == 1751195974.5656028, "unexpected last timestamp"
+    assert set(neo_io.multiplexed_channel_xml.keys()) == set(
+        [
+            "Headstage_AccelX",
+            "Headstage_AccelY",
+            "Headstage_AccelZ",
+            "Headstage_GyroX",
+            "Headstage_GyroY",
+            "Headstage_GyroZ",
+            "Headstage_MagX",
+            "Headstage_MagY",
+            "Headstage_MagZ",
+            "Controller_Ain1",
+        ]
+    )
+    assert neo_io._raw_memmap.shape == (433012, 54)