Merging code from Taylor's fork (lina-usc#198)

* Update readme
* Integrating Taylor's changes for HPC
* Adjusting the flags.
* Fix test for changed flag names.

Author: christian-oreilly

README.md

@@ -37,31 +37,37 @@ Please find the full documentation at
 
 ## ▶️ Running the pyLossless Pipeline
 Below is a minimal example that runs the pipeline one of MNE's sample files.  
-```
+```python
 import pylossless as ll 
 import mne
 fname = mne.datasets.sample.data_path() / 'MEG' / 'sample' /  'sample_audvis_raw.fif'
 raw = mne.io.read_raw_fif(fname, preload=True)
 
 config = ll.config.Config()
 config.load_default()
-config.save("my_project_ll_config.yaml")
 
-pipeline = ll.LosslessPipeline('my_project_ll_config.yaml')
+pipeline = ll.LosslessPipeline(config=config)
 pipeline.run_with_raw(raw)
 ```
 
 Once it is completed, You can see what channels and times were flagged:
-```
+```python
 print(pipeline.flagged_chs)
 print(pipeline.flagged_epochs)
 ```
 
-Once you are ready, you can save your file:
-```
+Once you are ready, you can save your file in its lossless state:
+```python
 pipeline.save(pipeline.get_derivative_path(bids_path), overwrite=True)
 ```
 
+To get a **cleaned** version, you can use a `RejectionPolicy` object to apply
+these annotations to your raw object. This is a lossy operation:
+```python
+rejection_policy = ll.RejectionPolicy()
+cleaned_raw = rejection_policy.apply(pipeline)
+```
+
 ## 👩‍💻 Dashboard Review
 [![Open in Colab](https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667)](https://colab.research.google.com/github/lina-usc/pylossless/blob/main/notebooks/qc_example.ipynb)
 
@@ -90,6 +96,7 @@ If you are a Canadian researcher working on an HPC system such as [Narval](https
 module load python/3.10
 
 # Build the virtualenv in your homedir
+cd ~
 virtualenv --no-download eeg-env
 source eeg-env/bin/activate
 
@@ -99,17 +106,12 @@ pip install --no-index xarray
 pip install --no-index pyyaml
 pip install --no-index sklearn
 pip install mne_bids
+pip install edfio
+pip install openneuro-py
 
-# Clone down mne-iclabel and switch to the right version and install it locally
-git clone https://github.com/mne-tools/mne-icalabel.git
-cd mne-icalabel
-git checkout maint/0.4
-pip install .
+pip install mne-icalabel
 
-# Clone down pipeline and install without reading dependencies
-git clone [email protected]:lina-usc/pylossless.git
-cd pylossless
-pip install --no-deps .
+pip install --no-deps pylossless
 
 # Verify that the package has installed correct with an import
 python -c 'import pylossless'

pylossless/assets/ll_default_config_adults.yaml

@@ -24,28 +24,6 @@ project:
     EEGCoordinateSystem: Other
     EEGCoordinateUnits: metres
 
-  general_info:
-    authors: [Unspecified]
-    institution_Name: Unspecified
-    institution_address: Unspecified
-    dataset_doi: []
-    funding: Unspecified
-    how_to_acknowledge: tba
-    license: ""
-    name: Unspecified
-    references_and_links: []
-
-  t_info:
-    EEG_placement_scheme: EGI 129
-    cap_manufacturer: EGI
-    cap_manufacturers_model_name: Hydrocel 129 Channel
-    hardware_filters: n/a
-    manufacturer: Electrical Geodesics
-    manufacturers_model_name: NetAmps300
-    power_line_frequency: 60
-    software_filters: n/a
-    software_versions: NetStation V4.5
-
 ######################## Task break detection ########################
 # See arguments definition from mne.preprocessing.annotate_breaks
 find_breaks:
@@ -69,19 +47,6 @@ filtering:
   notch_filter_args:
     freqs: [60]
 
-############################## SLURM #################################
-# Options for running the pipeline on a cluster
-# through SLURM
-slurm_options:
-  account: def-emayada
-  job_name: pylossless
-  memory: 16g
-  mpi: false
-  num_tasks: 1
-  program_options: []
-  threads_per_task: []
-  time_limit: 2h
-
 ########################## Nearest neighbor ##########################
 nearest_neighbors:
   n_nbr_ch: 3

pylossless/assets/ll_default_config_infants.yaml

@@ -24,28 +24,6 @@ project:
     EEGCoordinateSystem: Other
     EEGCoordinateUnits: metres
 
-  general_info:
-    authors: [Unspecified]
-    institution_Name: Unspecified
-    institution_address: Unspecified
-    dataset_doi: []
-    funding: Unspecified
-    how_to_acknowledge: tba
-    license: ""
-    name: Unspecified
-    references_and_links: []
-
-  t_info:
-    EEG_placement_scheme: EGI 129
-    cap_manufacturer: EGI
-    cap_manufacturers_model_name: Hydrocel 129 Channel
-    hardware_filters: n/a
-    manufacturer: Electrical Geodesics
-    manufacturers_model_name: NetAmps300
-    power_line_frequency: 60
-    software_filters: n/a
-    software_versions: NetStation V4.5
-
 ######################## Task break detection ########################
 # See arguments definition from mne.preprocessing.annotate_breaks
 find_breaks:
@@ -69,19 +47,6 @@ filtering:
   notch_filter_args:
     freqs: []
 
-############################## SLURM #################################
-# Options for running the pipeline on a cluster
-# through SLURM
-slurm_options:
-  account: def-emayada
-  job_name: pylossless
-  memory: 16g
-  mpi: false
-  num_tasks: 1
-  program_options: []
-  threads_per_task: []
-  time_limit: 2h
-
 ########################## Nearest neighbor ##########################
 nearest_neighbors:
   n_nbr_ch: 3

pylossless/assets/test_data/sub-s01/eeg/sub-s01_task-faceO_ll_config.yaml

@@ -92,38 +92,8 @@ project:
     coordsys:
         EEGCoordinateSystem: Other
         EEGCoordinateUnits: metres
-    general_info:
-        authors:
-        - Q1K Neuroimaging group
-        dataset_doi: []
-        funding: Azrieli Foundationt
-        how_to_acknowledge: tba
-        institution_Name: McGill University
-        institution_address: 3775 Rue University, Montreal, QC
-        license: ''
-        name: Q1K Mismatched Negativity
-        references_and_links: []
     readme: '# Q1K ACAR Dataset'
     set_montage_kwargs: {}
-    t_info:
-        EEG_placement_scheme: EGI 129
-        cap_manufacturer: EGI
-        cap_manufacturers_model_name: Hydrocel 129 Channel
-        hardware_filters: n/a
-        manufacturer: Electrical Geodesics
-        manufacturers_model_name: NetAmps300
-        power_line_frequency: 60
-        software_filters: n/a
-        software_versions: NetStation V4.5
 ref_loc_file: derivatives/EEG-IP-L/code/misc/standard_1020_ll_ref19.elc
 save_f_res: 1
-sd_t_pad: 1
-slurm_options:
-    account: def-emayada
-    job_name: pylossless
-    memory: 16g
-    mpi: false
-    num_tasks: 1
-    program_options: []
-    threads_per_task: []
-    time_limit: 2h
+sd_t_pad: 1

pylossless/assets/test_data/sub-s02/eeg/sub-s02_task-faceO_ll_config.yaml

@@ -92,38 +92,8 @@ project:
     coordsys:
         EEGCoordinateSystem: Other
         EEGCoordinateUnits: metres
-    general_info:
-        authors:
-        - Q1K Neuroimaging group
-        dataset_doi: []
-        funding: Azrieli Foundationt
-        how_to_acknowledge: tba
-        institution_Name: McGill University
-        institution_address: 3775 Rue University, Montreal, QC
-        license: ''
-        name: Q1K Mismatched Negativity
-        references_and_links: []
     readme: '# Q1K ACAR Dataset'
     set_montage_kwargs: {}
-    t_info:
-        EEG_placement_scheme: EGI 129
-        cap_manufacturer: EGI
-        cap_manufacturers_model_name: Hydrocel 129 Channel
-        hardware_filters: n/a
-        manufacturer: Electrical Geodesics
-        manufacturers_model_name: NetAmps300
-        power_line_frequency: 60
-        software_filters: n/a
-        software_versions: NetStation V4.5
 ref_loc_file: derivatives/EEG-IP-L/code/misc/standard_1020_ll_ref19.elc
 save_f_res: 1
 sd_t_pad: 1
-slurm_options:
-    account: def-emayada
-    job_name: pylossless
-    memory: 16g
-    mpi: false
-    num_tasks: 1
-    program_options: []
-    threads_per_task: []
-    time_limit: 2h

pylossless/flagging.py

@@ -18,38 +18,29 @@
 from .utils._utils import _icalabel_to_data_frame
 
 IC_LABELS = mne_icalabel.config.ICA_LABELS_TO_MNE
-CH_LABELS: dict[str, str] = {
-    "Noisy": "ch_sd",
-    "Bridged": "bridge",
-    "Uncorrelated": "low_r",
-    "Rank": "rank"
-}
-EPOCH_LABELS: dict[str, str] = {
-    "Noisy": "noisy",
-    "Noisy ICs": "noisy_ICs",
-    "Uncorrelated": "uncorrelated",
-}
+CH_LABELS: list = ["noisy", "bridged", "uncorrelated", "rank"]
+EPOCH_LABELS: list = ["noisy", "noisy_ICs", "uncorrelated"]
 
 
 class _Flagged(dict):
 
-    def __init__(self, key_map, kind_str, ll, *args, **kwargs):
+    def __init__(self, keys, kind_str, ll, *args, **kwargs):
         """Initialize class."""
         super().__init__(*args, **kwargs)
         self.ll = ll
-        self._key_map = key_map
+        self._keys = keys
         self._kind_str = kind_str
 
     @property
     def valid_keys(self):
         """Return the valid keys."""
-        return tuple(self._key_map.values())
+        return tuple(self._keys)
 
     def __repr__(self):
         """Return a string representation."""
         ret_str = f"Flagged {self._kind_str}s: |\n"
-        for key, val in self._key_map.items():
-            ret_str += f"  {key}: {self.get(val, None)}\n"
+        for key in self._keys:
+            ret_str += f"  {key.title().replace('_', ' ')}: {self.get(key, None)}\n"
         return ret_str
 
     def __eq__(self, other):
@@ -102,8 +93,7 @@ def add_flag_cat(self, kind, bad_ch_names, *args):
         Parameters
         ----------
         kind : str
-            Should be one of ``'ch_sd'``, ``'low_r'``,
-            ``'bridge'``, ``'rank'``.
+            Should be one of the values in ``CH_LABELS``.
         bad_ch_names : list | tuple
             Channel names. Will be the values corresponding to the ``kind``
             dictionary key.
@@ -220,7 +210,7 @@ def add_flag_cat(self, kind, bad_epoch_inds, epochs):
         Parameters
         ----------
         kind : str
-            Should be one of ``'noisy'``, ``'uncorrelated'``, ``'noisy_ICs'``.
+            Should be one of the values in ``EPOCH_LABELS``.
         bad_epochs_inds : list | tuple
             Indices for the epochs in an :class:`mne.Epochs` object. Will be
             the values for the ``kind`` dictionary key.

pylossless/pipeline.py

@@ -509,6 +509,7 @@ def _repr_html_(self):
             "Noisy": ch_flags.get("noisy", None),
             "Bridged": ch_flags.get("bridged", None),
             "Uncorrelated": ch_flags.get("uncorrelated", None),
+            "Rank": ch_flags.get("rank", None),
         }
         html += _create_html_details("Flagged Channels", flagged_channels_data)
 

pylossless/tests/test_pipeline.py

@@ -121,9 +121,9 @@ def test_load_flags(pipeline_fixture, tmp_path):
     pipeline = ll.LosslessPipeline().load_ll_derivative(bids_path)
 
     assert pipeline_fixture.flags['ch'] == pipeline.flags['ch']
-    pipeline.flags['ch']["bridge"] = ["xx"]
+    pipeline.flags['ch']["bridged"] = ["xx"]
     assert pipeline_fixture.flags['ch'] != pipeline.flags['ch']
 
     assert pipeline_fixture.flags['epoch'] == pipeline.flags['epoch']
-    pipeline.flags['epoch']["bridge"] = ["noisy"]
+    pipeline.flags['epoch']["bridged"] = ["noisy"]
     assert pipeline_fixture.flags['epoch'] == pipeline.flags['epoch']

requirements.txt

@@ -4,6 +4,8 @@ mne_bids>=0.14
 pandas
 xarray
 scipy>=1.7.1
-mne_icalabel>=0.5.0
+mne_icalabel>=0.6.0
 pyyaml
 scikit-learn
+torch
+edfio

setup.py

@@ -43,4 +43,5 @@
     extras_require=extras_require,
     include_package_data=True,
     entry_points={"console_scripts": qc_entry_point},
+    dependency_links=['https://download.pytorch.org/whl/cpu'],
 )