From fa17de9730516d089924d8cf4e0bfe4b7dd93378 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Wed, 17 Sep 2025 16:10:02 -0700 Subject: [PATCH 01/16] convert elecrtricalseries to uV, rename impedance column to imp --- src/jdb_to_nwb/convert_raw_ephys.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index b15ac61..3a4aac8 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -28,9 +28,6 @@ from .plotting.plot_ephys import plot_channel_map, plot_channel_impedances, plot_neuropixels from ndx_franklab_novela import AssociatedFiles, Probe, NwbElectrodeGroup, Shank, ShanksElectrode -MICROVOLTS_PER_VOLT = 1e6 -VOLTS_PER_MICROVOLT = 1 / MICROVOLTS_PER_VOLT - MIN_IMPEDANCE_OHMS = 1e5 MAX_IMPEDANCE_OHMS = 3e6 @@ -248,7 +245,7 @@ def get_raw_ephys_data( traces_as_iterator (SpikeInterfaceRecordingDataChunkIterator): To be used as the data argument in pynwb.ecephys.ElectricalSeries. channel_conversion_factor (float): - The conversion factor from the raw data to volts. + The conversion factor from the raw data to uV. original_timestamps (np.ndarray): Array that could be used as the timestamps argument in pynwb.ecephys.ElectricalSeries or may need to be time aligned with the other data streams in the NWB file. @@ -298,8 +295,9 @@ def get_raw_ephys_data( "The channel conversion factors are not the same for all channels. " "This is unexpected and may indicate a problem with the conversion factors." ) - channel_conversion_factor_v = channel_conversion_factors_uv[0] * VOLTS_PER_MICROVOLT - logger.debug(f"Channel conversion factor in V: {channel_conversion_factor_v}") + # Just grab the first one, because it should be the same for all channels + channel_conversion_factor_uv = channel_conversion_factors_uv[0] + logger.debug(f"Channel conversion factor in uV: {channel_conversion_factor_uv}") # NOTE channel offsets should be 0 for all channels in openephys data channel_conversion_offsets = recording_sliced.get_channel_offsets() @@ -317,7 +315,7 @@ def get_raw_ephys_data( return ( traces_as_iterator, - channel_conversion_factor_v, + channel_conversion_factor_uv, original_timestamps, ) @@ -750,7 +748,7 @@ def add_electrode_data_berke_probe( description="The port of the electrode from the impedance file", ) nwbfile.add_electrode_column( - name="impedance", + name="imp", description="The impedance of the electrode (Impedance Magnitude at 1000 Hz (ohms))", ) nwbfile.add_electrode_column( @@ -847,7 +845,7 @@ def add_electrode_data_berke_probe( open_ephys_channel_str=row["open_ephys_channel_string"], channel_name=row["Channel Name"], port=row["Port"], - impedance=row["Impedance Magnitude at 1000 Hz (ohms)"], + imp=row["Impedance Magnitude at 1000 Hz (ohms)"], imp_phase=row["Impedance Phase at 1000 Hz (degrees)"], series_resistance_in_ohms=row["Series RC equivalent R (Ohms)"], series_capacitance_in_farads=row["Series RC equivalent C (Farads)"], @@ -1129,7 +1127,7 @@ def add_raw_ephys( # Get raw ephys data ( traces_as_iterator, - channel_conversion_factor_v, + channel_conversion_factor_uv, original_timestamps, ) = get_raw_ephys_data(folder_path=openephys_folder_path, logger=logger, exclude_channels=exclude_channel_names) num_samples, num_channels = traces_as_iterator.maxshape @@ -1157,6 +1155,11 @@ def add_raw_ephys( region=list(range(len(nwbfile.electrodes))), description="Electrodes used in raw ElectricalSeries recording", ) + + # Convert to uV without loading the whole thing at once + def traces_in_uV_iterator(traces_as_iterator, conversion_factor): + for chunk in traces_as_iterator: + yield chunk.astype(np.float32) * conversion_factor # A chunk of shape (81920, 64) and dtype int16 (2 bytes) is ~10 MB, which is the recommended chunk size # by the NWB team. @@ -1164,7 +1167,7 @@ def add_raw_ephys( # they require the hdf5plugin library to be installed. gzip is available by default. # Use gzip for now, but consider zstd/blosc-zstd in the future. data_data_io = H5DataIO( - traces_as_iterator, + traces_in_uV_iterator(traces_as_iterator, channel_conversion_factor_uv), chunks=(min(num_samples, 81920), min(num_channels, 64)), compression="gzip", ) @@ -1197,7 +1200,8 @@ def add_raw_ephys( data=data_data_io, timestamps=ephys_timestamps, electrodes=electrode_table_region, - conversion=channel_conversion_factor_v, + conversion=1.0, + unit="microvolts", ) # Add the ElectricalSeries to the NWBFile From 178e097148f6d64220d312cebe350afa3100ba62 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Wed, 17 Sep 2025 19:42:27 -0700 Subject: [PATCH 02/16] add conversion to volts --- src/jdb_to_nwb/convert_raw_ephys.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index 3a4aac8..66310bf 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -1196,12 +1196,12 @@ def traces_in_uV_iterator(traces_as_iterator, conversion_factor): # For now, we do not chunk or compress the timestamps, which are relatively small eseries = ElectricalSeries( name="ElectricalSeries", - description="Raw ephys data from OpenEphys recording (multiply by conversion factor to get data in volts).", + description="Raw ephys data from OpenEphys recording, in uV (multiply by conversion factor to get data in V).", data=data_data_io, timestamps=ephys_timestamps, electrodes=electrode_table_region, - conversion=1.0, - unit="microvolts", + conversion=1/1e6, # conversion from uV to V + unit="volts", ) # Add the ElectricalSeries to the NWBFile From 606380be3527a85dd6adf7378e91a6a41fce15e0 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Tue, 23 Sep 2025 16:33:22 +0200 Subject: [PATCH 03/16] update ephys tests to reflect changes, maybe fix ephys data iterator? --- src/jdb_to_nwb/convert_raw_ephys.py | 15 +++++++++++---- tests/test_convert_raw_ephys.py | 20 ++++++++++---------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index 66310bf..2f7d585 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -21,6 +21,7 @@ ) from pynwb import NWBFile from pynwb.ecephys import ElectricalSeries +from hdmf.data_utils import DataChunkIterator from spikeinterface.extractors import OpenEphysBinaryRecordingExtractor from .utils import get_logger_directory @@ -1155,11 +1156,17 @@ def add_raw_ephys( region=list(range(len(nwbfile.electrodes))), description="Electrodes used in raw ElectricalSeries recording", ) - + # Convert to uV without loading the whole thing at once - def traces_in_uV_iterator(traces_as_iterator, conversion_factor): + def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): for chunk in traces_as_iterator: - yield chunk.astype(np.float32) * conversion_factor + yield chunk.astype(np.float32) * conversion_factor_uv + + # Wrap iterator in DataChunkIterator for H5DataIO + data_iterator = DataChunkIterator( + traces_in_microvolts_iterator(traces_as_iterator, channel_conversion_factor_uv), + buffer_size=1 # number of chunks to keep in memory + ) # A chunk of shape (81920, 64) and dtype int16 (2 bytes) is ~10 MB, which is the recommended chunk size # by the NWB team. @@ -1167,7 +1174,7 @@ def traces_in_uV_iterator(traces_as_iterator, conversion_factor): # they require the hdf5plugin library to be installed. gzip is available by default. # Use gzip for now, but consider zstd/blosc-zstd in the future. data_data_io = H5DataIO( - traces_in_uV_iterator(traces_as_iterator, channel_conversion_factor_uv), + data_iterator(traces_as_iterator, channel_conversion_factor_uv), chunks=(min(num_samples, 81920), min(num_channels, 64)), compression="gzip", ) diff --git a/tests/test_convert_raw_ephys.py b/tests/test_convert_raw_ephys.py index 43a8cc3..9ebc3a5 100644 --- a/tests/test_convert_raw_ephys.py +++ b/tests/test_convert_raw_ephys.py @@ -83,7 +83,7 @@ def test_add_electrode_data_berke_probe(dummy_logger): metadata["ephys"]["electrodes_location"] = "Hippocampus CA1" metadata["ephys"]["targeted_x"] = 4.5 # AP in mm metadata["ephys"]["targeted_y"] = 2.2 # ML in mm - metadata["ephys"]["targeted_z"] = 2.0 # DV in mm + metadata["ephys"]["targeted_z"] = -2.0 # DV in mm metadata["ephys"]["probe"] = ["256-ch Silicon Probe, 3mm length, 66um pitch"] metadata["ephys"]["plug_order"] = "chip_first" @@ -146,7 +146,7 @@ def test_add_electrode_data_berke_probe(dummy_logger): assert egroup.location == "Hippocampus CA1" assert egroup.targeted_x == 4.5 assert egroup.targeted_y == 2.2 - assert egroup.targeted_z == 2.0 + assert egroup.targeted_z == -2.0 assert egroup.device is probe # Electrode group description should be "Electrodes on shank {shank_index}" @@ -174,7 +174,7 @@ def test_add_electrode_data_berke_probe(dummy_logger): assert (nwbfile.electrodes.intan_channel_number.data[:] == expected_intan_channel_numbers).all() # Check first electrode data (S01E1) - assert nwbfile.electrodes.impedance.data[0] == 6.50e06 + assert nwbfile.electrodes.imp.data[0] == 6.50e06 assert nwbfile.electrodes.imp_phase.data[0] == -68 assert nwbfile.electrodes.series_resistance_in_ohms.data[0] == 2.44e06 assert nwbfile.electrodes.series_capacitance_in_farads.data[0] == 2.64e-11 @@ -183,7 +183,7 @@ def test_add_electrode_data_berke_probe(dummy_logger): assert nwbfile.electrodes.rel_y.data[0] == 255.0 # Check last electrode data (S32E8) - assert nwbfile.electrodes.impedance.data[-1] == 2.24e06 + assert nwbfile.electrodes.imp.data[-1] == 2.24e06 assert nwbfile.electrodes.imp_phase.data[-1] == -43 assert nwbfile.electrodes.series_resistance_in_ohms.data[-1] == 1.63e06 assert nwbfile.electrodes.series_capacitance_in_farads.data[-1] == 1.04e-10 @@ -209,7 +209,7 @@ def test_get_raw_ephys_data(dummy_logger): folder_path = "tests/test_data/raw_ephys/2022-07-25_15-30-00" traces_as_iterator, channel_conversion_factor, original_timestamps = get_raw_ephys_data(folder_path, dummy_logger) assert traces_as_iterator.maxshape == (3_000, 256) - np.testing.assert_allclose(channel_conversion_factor, [0.19499999284744263 * 1e-6] * 256) + np.testing.assert_allclose(channel_conversion_factor, [0.19499999284744263] * 256) assert len(original_timestamps) == 3_000 @@ -252,7 +252,7 @@ def test_add_raw_ephys(dummy_logger): metadata["ephys"]["electrodes_location"] = "Hippocampus CA1" metadata["ephys"]["targeted_x"] = 4.5 # AP in mm metadata["ephys"]["targeted_y"] = 2.2 # ML in mm - metadata["ephys"]["targeted_z"] = 2.0 # DV in mm + metadata["ephys"]["targeted_z"] = -2.0 # DV in mm metadata["ephys"]["probe"] = ["256-ch Silicon Probe, 3mm length, 66um pitch"] ephys_data_dict = add_raw_ephys(nwbfile=nwbfile, metadata=metadata, logger=dummy_logger) @@ -263,13 +263,13 @@ def test_add_raw_ephys(dummy_logger): assert "ElectricalSeries" in nwbfile.acquisition es = nwbfile.acquisition["ElectricalSeries"] assert es.description == ( - "Raw ephys data from OpenEphys recording (multiply by conversion factor to get data in volts)." + "Raw ephys data from OpenEphys recording, in uV (multiply by conversion factor to get data in V)." ) assert es.data.maxshape == (3_000, 256) assert es.data.dtype == np.int16 assert es.electrodes.data == list(range(256)) assert es.timestamps.shape == (3_000,) - assert es.conversion == 0.19499999284744263 * 1e-6 + assert es.conversion == 1/1e-6 # Test that the nwbfile has the expected associated files assert "associated_files" in nwbfile.processing @@ -356,9 +356,9 @@ def test_add_raw_ephys_complete_data(): metadata["ephys"]["electrodes_location"] = "Hippocampus CA1" metadata["ephys"]["targeted_x"] = 4.5 # AP in mm metadata["ephys"]["targeted_y"] = 2.2 # ML in mm - metadata["ephys"]["targeted_z"] = 2.0 # DV in mm + metadata["ephys"]["targeted_z"] = -2.0 # DV in mm metadata["ephys"]["probe"] = ["256-ch Silicon Probe, 3mm length, 66um pitch"] - + ephys_data_dict = add_raw_ephys(nwbfile=nwbfile, metadata=metadata) assert len(nwbfile.electrodes) == 256 From 056aa95994d16cfae61e5d0a7db11fca8270cee5 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Tue, 23 Sep 2025 16:45:21 +0200 Subject: [PATCH 04/16] sillie fix, oopsie! so glad im publicly testing my code on GHA <3 --- src/jdb_to_nwb/convert_raw_ephys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index 2f7d585..29c3b30 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -1174,7 +1174,7 @@ def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): # they require the hdf5plugin library to be installed. gzip is available by default. # Use gzip for now, but consider zstd/blosc-zstd in the future. data_data_io = H5DataIO( - data_iterator(traces_as_iterator, channel_conversion_factor_uv), + data_iterator, chunks=(min(num_samples, 81920), min(num_channels, 64)), compression="gzip", ) From a824863355d29c00221141beddf7da0a7d15f324 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Tue, 23 Sep 2025 17:00:06 +0200 Subject: [PATCH 05/16] remove 'unit' field from ElectricalSeries, it doesn't exist as nwb always expects conversion is to volts --- src/jdb_to_nwb/convert_raw_ephys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index 29c3b30..04a914e 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -1208,7 +1208,6 @@ def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): timestamps=ephys_timestamps, electrodes=electrode_table_region, conversion=1/1e6, # conversion from uV to V - unit="volts", ) # Add the ElectricalSeries to the NWBFile From 7ccc0fa09bd9c187340a85d5aee2e953bb016775 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Tue, 23 Sep 2025 17:26:52 +0200 Subject: [PATCH 06/16] try to force 2d? kinda messy idk --- src/jdb_to_nwb/convert_raw_ephys.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index 04a914e..bab7f8c 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -1160,11 +1160,13 @@ def add_raw_ephys( # Convert to uV without loading the whole thing at once def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): for chunk in traces_as_iterator: - yield chunk.astype(np.float32) * conversion_factor_uv + arr = np.asarray(chunk) + arr = arr.reshape(-1, num_channels) # force 2D + yield arr.astype(np.float32) * conversion_factor_uv # Wrap iterator in DataChunkIterator for H5DataIO data_iterator = DataChunkIterator( - traces_in_microvolts_iterator(traces_as_iterator, channel_conversion_factor_uv), + traces_in_microvolts_iterator(traces_as_iterator, channel_conversion_factor_uv, num_channels), buffer_size=1 # number of chunks to keep in memory ) From b022c59f88ed4d4fc07fc1940eb989c1bb75d8f6 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Tue, 23 Sep 2025 17:39:14 +0200 Subject: [PATCH 07/16] lol --- src/jdb_to_nwb/convert_raw_ephys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index bab7f8c..5a2a919 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -1158,7 +1158,7 @@ def add_raw_ephys( ) # Convert to uV without loading the whole thing at once - def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): + def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv, num_channels): for chunk in traces_as_iterator: arr = np.asarray(chunk) arr = arr.reshape(-1, num_channels) # force 2D From 0e39487f3e52e305484af15f86705414064c50a9 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Tue, 23 Sep 2025 17:55:06 +0200 Subject: [PATCH 08/16] just telling it the shape is maybe better? --- src/jdb_to_nwb/convert_raw_ephys.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index 5a2a919..26eb451 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -1158,16 +1158,16 @@ def add_raw_ephys( ) # Convert to uV without loading the whole thing at once - def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv, num_channels): + def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): for chunk in traces_as_iterator: - arr = np.asarray(chunk) - arr = arr.reshape(-1, num_channels) # force 2D - yield arr.astype(np.float32) * conversion_factor_uv + yield chunk.astype(np.float32) * conversion_factor_uv # Wrap iterator in DataChunkIterator for H5DataIO data_iterator = DataChunkIterator( traces_in_microvolts_iterator(traces_as_iterator, channel_conversion_factor_uv, num_channels), - buffer_size=1 # number of chunks to keep in memory + buffer_size=1, # number of chunks to keep in memory + data_shape=(num_samples, num_channels), + dtype=np.float32, ) # A chunk of shape (81920, 64) and dtype int16 (2 bytes) is ~10 MB, which is the recommended chunk size From e115149772b51f79fab6ae2b9c9d9265527eddd3 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Tue, 23 Sep 2025 18:05:49 +0200 Subject: [PATCH 09/16] every time i change the function i forget to change the call sry if u get emails when i rerun test --- src/jdb_to_nwb/convert_raw_ephys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index 26eb451..62d4ec2 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -1164,7 +1164,7 @@ def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): # Wrap iterator in DataChunkIterator for H5DataIO data_iterator = DataChunkIterator( - traces_in_microvolts_iterator(traces_as_iterator, channel_conversion_factor_uv, num_channels), + traces_in_microvolts_iterator(traces_as_iterator, channel_conversion_factor_uv), buffer_size=1, # number of chunks to keep in memory data_shape=(num_samples, num_channels), dtype=np.float32, From 8adcbbccf55b10e26baa471267a60f4a2dd30f15 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Tue, 23 Sep 2025 18:24:29 +0200 Subject: [PATCH 10/16] hm --- src/jdb_to_nwb/convert_raw_ephys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index 62d4ec2..f3579ae 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -1166,7 +1166,7 @@ def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): data_iterator = DataChunkIterator( traces_in_microvolts_iterator(traces_as_iterator, channel_conversion_factor_uv), buffer_size=1, # number of chunks to keep in memory - data_shape=(num_samples, num_channels), + maxshape=(num_samples, num_channels), dtype=np.float32, ) From 84e78f421d867483a13d8e659709009d5ce0dd74 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Tue, 23 Sep 2025 21:26:32 +0200 Subject: [PATCH 11/16] dtype?? --- src/jdb_to_nwb/convert_raw_ephys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index f3579ae..6540944 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -1167,7 +1167,7 @@ def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): traces_in_microvolts_iterator(traces_as_iterator, channel_conversion_factor_uv), buffer_size=1, # number of chunks to keep in memory maxshape=(num_samples, num_channels), - dtype=np.float32, + dtype=np.dtype("float32"), ) # A chunk of shape (81920, 64) and dtype int16 (2 bytes) is ~10 MB, which is the recommended chunk size From 06d98f88f8e1a848ac8a68d11dcb5b9360ac0457 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Tue, 23 Sep 2025 21:58:39 +0200 Subject: [PATCH 12/16] test fix real quick --- tests/test_convert_raw_ephys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_convert_raw_ephys.py b/tests/test_convert_raw_ephys.py index 9ebc3a5..98b3dd0 100644 --- a/tests/test_convert_raw_ephys.py +++ b/tests/test_convert_raw_ephys.py @@ -266,7 +266,7 @@ def test_add_raw_ephys(dummy_logger): "Raw ephys data from OpenEphys recording, in uV (multiply by conversion factor to get data in V)." ) assert es.data.maxshape == (3_000, 256) - assert es.data.dtype == np.int16 + # assert es.data.dtype == np.int16 assert es.electrodes.data == list(range(256)) assert es.timestamps.shape == (3_000,) assert es.conversion == 1/1e-6 From 04cfc52804d95f4632eac343b956dcab74db2499 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Wed, 24 Sep 2025 09:49:49 +0200 Subject: [PATCH 13/16] got silly w my exponents, fixed it --- src/jdb_to_nwb/convert_raw_ephys.py | 2 +- tests/test_convert_raw_ephys.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index 6540944..8f50f67 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -1209,7 +1209,7 @@ def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): data=data_data_io, timestamps=ephys_timestamps, electrodes=electrode_table_region, - conversion=1/1e6, # conversion from uV to V + conversion=1e-6, # conversion from uV to V ) # Add the ElectricalSeries to the NWBFile diff --git a/tests/test_convert_raw_ephys.py b/tests/test_convert_raw_ephys.py index 98b3dd0..2186370 100644 --- a/tests/test_convert_raw_ephys.py +++ b/tests/test_convert_raw_ephys.py @@ -269,7 +269,7 @@ def test_add_raw_ephys(dummy_logger): # assert es.data.dtype == np.int16 assert es.electrodes.data == list(range(256)) assert es.timestamps.shape == (3_000,) - assert es.conversion == 1/1e-6 + assert es.conversion == 1e-6 # Test that the nwbfile has the expected associated files assert "associated_files" in nwbfile.processing From 2f40de44f375b242b0429bc2509afc0a53ce299b Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Wed, 24 Sep 2025 10:06:24 +0200 Subject: [PATCH 14/16] is float32 gonna be way too big? --- tests/test_convert_raw_ephys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_convert_raw_ephys.py b/tests/test_convert_raw_ephys.py index 2186370..95bb9e0 100644 --- a/tests/test_convert_raw_ephys.py +++ b/tests/test_convert_raw_ephys.py @@ -266,7 +266,7 @@ def test_add_raw_ephys(dummy_logger): "Raw ephys data from OpenEphys recording, in uV (multiply by conversion factor to get data in V)." ) assert es.data.maxshape == (3_000, 256) - # assert es.data.dtype == np.int16 + assert es.data.dtype == np.float32 assert es.electrodes.data == list(range(256)) assert es.timestamps.shape == (3_000,) assert es.conversion == 1e-6 From 3b042fde62c788da43d6d01eb522f4e2bb52f119 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Mon, 27 Oct 2025 10:40:53 -0700 Subject: [PATCH 15/16] store electricalseries in uv as int16 --- src/jdb_to_nwb/convert_raw_ephys.py | 4 ++-- tests/test_convert_raw_ephys.py | 23 +++++++++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index f76bb50..7dd593e 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -1167,14 +1167,14 @@ def add_raw_ephys( # Convert to uV without loading the whole thing at once def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): for chunk in traces_as_iterator: - yield chunk.astype(np.float32) * conversion_factor_uv + yield (chunk * conversion_factor_uv).astype("int16") # Wrap iterator in DataChunkIterator for H5DataIO data_iterator = DataChunkIterator( traces_in_microvolts_iterator(traces_as_iterator, channel_conversion_factor_uv), buffer_size=1, # number of chunks to keep in memory maxshape=(num_samples, num_channels), - dtype=np.dtype("float32"), + dtype=np.int16, ) # A chunk of shape (81920, 64) and dtype int16 (2 bytes) is ~10 MB, which is the recommended chunk size diff --git a/tests/test_convert_raw_ephys.py b/tests/test_convert_raw_ephys.py index d6fb3a3..d64728e 100644 --- a/tests/test_convert_raw_ephys.py +++ b/tests/test_convert_raw_ephys.py @@ -255,7 +255,7 @@ def test_add_raw_ephys(dummy_logger): metadata["ephys"]["targeted_z"] = -2.0 # DV in mm metadata["ephys"]["probe"] = ["256-ch Silicon Probe, 3mm length, 66um pitch"] - # Mapping from probe electrode to channel number + # Mapping from probe electrode to channel number for 256-ch silicon probe intan_channel_numbers = np.array([ 191,190,189,188,187,186,185,184,183,182,181,180,179,178,177,176,175,174,173,172,171,170,169,128,129,130, 131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156, @@ -373,6 +373,21 @@ def test_add_raw_ephys_complete_data(): metadata["ephys"]["targeted_y"] = 2.2 # ML in mm metadata["ephys"]["targeted_z"] = -2.0 # DV in mm metadata["ephys"]["probe"] = ["256-ch Silicon Probe, 3mm length, 66um pitch"] + + # Mapping from probe electrode to channel number for 256-ch silicon probe + intan_channel_numbers = np.array([ + 191,190,189,188,187,186,185,184,183,182,181,180,179,178,177,176,175,174,173,172,171,170,169,128,129,130, + 131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156, + 157,158,159,160,161,162,163,164,165,166,167,168,192,193,194,195,196,197,198,199,200,201,202,203,204,205, + 206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231, + 232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,64,65,66, + 67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100, + 101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126, + 127,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55, + 56,57,58,59,60,61,62,63,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 + ]) + # Reverse mapping from channel (aka row in ElectricalSeries) to row in electrode table + expected_electrode_data_mapping = list(np.argsort(intan_channel_numbers)) ephys_data_dict = add_raw_ephys(nwbfile=nwbfile, metadata=metadata) @@ -382,13 +397,13 @@ def test_add_raw_ephys_complete_data(): assert "ElectricalSeries" in nwbfile.acquisition es = nwbfile.acquisition["ElectricalSeries"] assert es.description == ( - "Raw ephys data from OpenEphys recording (multiply by conversion factor to get data in volts)." + "Raw ephys data from OpenEphys recording, in uV (multiply by conversion factor to get data in V)." ) assert es.data.maxshape == (157_733_308, 256) assert es.data.dtype == np.int16 - assert es.electrodes.data == list(range(256)) + assert es.electrodes.data == expected_electrode_data_mapping assert es.timestamps.shape == (157_733_308,) - assert es.conversion == 0.19499999284744263 * 1e-6 + assert es.conversion == 1e-6 # Test that the nwbfile has the expected associated files assert "associated_files" in nwbfile.processing From 5764bc82d466d2f5103dee2381fbc4e1cf9392e4 Mon Sep 17 00:00:00 2001 From: Stephanie Crater Date: Mon, 27 Oct 2025 10:54:38 -0700 Subject: [PATCH 16/16] fix dtype --- src/jdb_to_nwb/convert_raw_ephys.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/jdb_to_nwb/convert_raw_ephys.py b/src/jdb_to_nwb/convert_raw_ephys.py index 7dd593e..02f4b78 100644 --- a/src/jdb_to_nwb/convert_raw_ephys.py +++ b/src/jdb_to_nwb/convert_raw_ephys.py @@ -29,6 +29,8 @@ from .plotting.plot_ephys import plot_channel_map, plot_channel_impedances, plot_neuropixels from ndx_franklab_novela import AssociatedFiles, Probe, NwbElectrodeGroup, Shank, ShanksElectrode +VOLTS_PER_MICROVOLT = 1e-6 + MIN_IMPEDANCE_OHMS = 1e5 MAX_IMPEDANCE_OHMS = 3e6 @@ -1174,7 +1176,7 @@ def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): traces_in_microvolts_iterator(traces_as_iterator, channel_conversion_factor_uv), buffer_size=1, # number of chunks to keep in memory maxshape=(num_samples, num_channels), - dtype=np.int16, + dtype=np.dtype("int16"), ) # A chunk of shape (81920, 64) and dtype int16 (2 bytes) is ~10 MB, which is the recommended chunk size @@ -1216,7 +1218,7 @@ def traces_in_microvolts_iterator(traces_as_iterator, conversion_factor_uv): data=data_data_io, timestamps=ephys_timestamps, electrodes=electrode_table_region, - conversion=1e-6, # conversion from uV to V + conversion=VOLTS_PER_MICROVOLT, # conversion from uV to V ) # Add the ElectricalSeries to the NWBFile