Merge pull request #262 from pennmem/sess_num

cmlreaders version 0.10.5

Author: hherrema

CHANGELOG.rst

@@ -1,5 +1,13 @@
 Changes
 =======
+Version 0.10.5
+-------------
+**2024-1-24**
+
+* Added system 4 files to PathFinder, including `archived_eeg` which points to replaced elemem eeg
+* Loading events and EGG will automatically modify ``session`` field to match 'session' in data index,
+  fixing issues where events dataframe contains 'original session' value
+  
 Version 0.10.4
 -------------
 **2023-8-1**

cmlreaders/__init__.py

@@ -8,6 +8,6 @@
 from .readers import *  # noqa
 from .cmlreader import CMLReader  # noqa
 
-__version__ = "0.10.4"
+__version__ = "0.10.5"
 version_info = namedtuple("VersionInfo", "major,minor,patch")(
     *__version__.split('.'))

cmlreaders/constants.py

@@ -128,21 +128,33 @@
     ],
     "session_json": [
         "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/session.json",
+        "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/session.jsonl",
     ],
     "ramulator_session_folder": [
         "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/host_pc/*",
         "protocols/r1/subjects/{subject}/experiments/{experiment}/sessions/{session}/ephys/current_source/host_pc/*",
     ],
+    # Elemem-related information
+    "elemem_session_folder": [
+        "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/elemem/*",
+        "protocols/r1/subjects/{subject}/experiments/{experiment}/sessions/{session}/ephys/current_source/elemem/*",
+    ],
     # There can be multiple timestamped folders for the host pc files for when
     # a session is restarted
     "event_log": [
         "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/host_pc/{timestamped_dir}/event_log.json",
+        "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/elemem/{timestamped_dir}/event.log",
     ],
     "experiment_config": [
         "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/host_pc/{timestamped_dir}/experiment_config.json",
+        "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/elemem/{timestamped_dir}/experiment_config.json",
     ],
     "raw_eeg": [
         "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/host_pc/{timestamped_dir}/eeg_timeseries.h5",
+        "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/elemem/{timestamped_dir}/eeg_data.edf",
+    ],
+    "archived_eeg": [
+        "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/elemem/{timestamped_dir}/eeg_archive/eeg_data.edf",
     ],
     "odin_config": [
         "data10/RAM/subjects/{subject}/behavioral/{experiment}/session_{session}/host_pc/{timestamped_dir}/config_files/{subject}_*.csv",
@@ -229,3 +241,12 @@
 
 # All Ramulator files/directories
 ramulator_files = host_pc_files + used_classifier_files + ("ramulator_session_folder",)
+
+# All Elemem files/directories
+elemem_files = (
+    "event_log",
+    "experiment_config",
+    "raw_eeg",
+    "elemem_session_folder",
+    "archived_eeg"
+)

cmlreaders/path_finder.py

@@ -7,7 +7,7 @@
 from typing import Optional
 
 from .constants import rhino_paths, localization_files, montage_files, \
-    subject_files, session_files, ramulator_files, \
+    subject_files, session_files, ramulator_files, elemem_files, \
     PYFR_SUBJECT_CODE_PREFIXES
 from .util import get_root_dir
 # from .warnings import MultiplePathsFoundWarning
@@ -80,6 +80,8 @@ def __init__(self, subject: Optional[str] = None,
 
         self._paths = rhino_paths
 
+        self.system_version = self._determine_system_version()
+
     @property
     def path_info(self):
         return self._paths
@@ -109,6 +111,21 @@ def subject_files(self):
         """ All files that vary only by subject """
         return subject_files
 
+    def _determine_system_version(self):
+        elemem_wildcard = self._paths["elemem_session_folder"][0].rstrip('*')
+
+        subject_montage = self.subject
+        if self.montage != '0':
+            subject_montage = "_".join([self.subject, self.montage])
+
+        elemem_path = elemem_wildcard.format(subject=subject_montage,
+                                             experiment=self.experiment,
+                                             session=self.session)
+        if os.path.exists(os.path.join(self.rootdir, elemem_path)):
+            return 4.0
+        else:
+            return 3.0  # only 4.0 is matched on, but may want to fully implement
+
     def find(self, data_type):
         """
 
@@ -148,18 +165,29 @@ def _lookup_file(self, data_type):
         timestamped_dir = None
 
         # Only check the host_pc folder if necessary
-        if data_type in ramulator_files:
-            folder_wildcard = self._paths['ramulator_session_folder'][0]
-            ramulator_session_folder = folder_wildcard.format(
-                subject=subject_montage, experiment=self.experiment,
-                session=self.session)
-
-            timestamped_dir = self._get_most_recent_ramulator_folder(
-                ramulator_session_folder)
-
-            # The user can also just request the folder
-            if data_type == 'ramulator_session_folder':
-                return timestamped_dir
+        if data_type in ramulator_files or data_type in elemem_files:
+            if self.system_version == 4.0:       # system 4
+                folder_wildcard = self._paths['elemem_session_folder'][0]
+                elemem_session_folder = folder_wildcard.format(
+                    subject=subject_montage, experiment=self.experiment,
+                    session=self.session)
+
+                timestamped_dir = self._get_most_recent_elemem_folder(elemem_session_folder)
+
+                if data_type == 'elemem_session_folder':
+                    return timestamped_dir
+            else:                     # system 3 (don't think will work for systems 1 and 2)
+                folder_wildcard = self._paths['ramulator_session_folder'][0]
+                ramulator_session_folder = folder_wildcard.format(
+                    subject=subject_montage, experiment=self.experiment,
+                    session=self.session)
+
+                timestamped_dir = self._get_most_recent_ramulator_folder(
+                    ramulator_session_folder)
+
+                # The user can also just request the folder
+                if data_type == 'ramulator_session_folder':
+                    return timestamped_dir
 
         expected_path = self._find_single_path(paths_to_check,
                                                protocol=self.protocol,
@@ -203,6 +231,27 @@ def _get_most_recent_ramulator_folder(self, base_folder_path):
 
         return latest_directory
 
+    def _get_most_recent_elemem_folder(self, base_folder_path):
+        timestamped_directories = glob.glob(os.path.join(self.rootdir, base_folder_path))
+
+        # start of directory should be subject code
+        timestamped_directories = [
+            d for d in timestamped_directories
+            if os.path.isdir(d) and os.path.basename(d)[:len(self.subject)] == self.subject
+        ]
+
+        # sort such that most recent appears first
+        timestamped_directories = sorted(timestamped_directories)[::-1]
+
+        if len(timestamped_directories) == 0:
+            raise RuntimeError("No timestamped folder found in elemem folder.")
+
+        # only return the values from the final "/" to the end
+        latest = timestamped_directories[0]
+        latest_directory = latest[latest.rfind("/") + 1:]
+
+        return latest_directory
+
     def _find_single_path(self, paths, **kwargs):
         final_paths = []
         for path in paths:

cmlreaders/readers/readers.py

@@ -243,6 +243,14 @@ def as_dataframe(self):
 
         first = ["eegoffset"]
         df = df[first + [col for col in df.columns if col not in first]]
+
+        # ensure session field matches data index
+        # math events given session, other events given original session
+        if df['session'].unique()[0] != self.session or len(df['session'].unique()) < 1:
+            warnings.warn(f'Changing events session field from {df["session"].unique()[0]} ' +
+                          f'to {self.session} to match data index.')
+            df['session'] = self.session
+
         return df