Integrating Taylor's changes for HPC

Author: christian-oreilly

README.md

@@ -37,7 +37,7 @@ Please find the full documentation at
 
 ## ▶️ Running the pyLossless Pipeline
 Below is a minimal example that runs the pipeline one of MNE's sample files.  
-```
+```python
 import pylossless as ll 
 import mne
 fname = mne.datasets.sample.data_path() / 'MEG' / 'sample' /  'sample_audvis_raw.fif'
@@ -51,16 +51,23 @@ pipeline.run_with_raw(raw)
 ```
 
 Once it is completed, You can see what channels and times were flagged:
-```
+```python
 print(pipeline.flagged_chs)
 print(pipeline.flagged_epochs)
 ```
 
-Once you are ready, you can save your file:
-```
+Once you are ready, you can save your file in its lossless state:
+```python
 pipeline.save(pipeline.get_derivative_path(bids_path), overwrite=True)
 ```
 
+To get a **cleaned** version, you can use a `RejectionPolicy` object to apply
+these annotations to your raw object. This is a lossy operation:
+```python
+rejection_policy = ll.RejectionPolicy()
+cleaned_raw = rejection_policy.apply(pipeline)
+```
+
 ## 👩‍💻 Dashboard Review
 [![Open in Colab](https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667)](https://colab.research.google.com/github/lina-usc/pylossless/blob/main/notebooks/qc_example.ipynb)
 
@@ -99,7 +106,7 @@ pip install --no-index xarray
 pip install --no-index pyyaml
 pip install --no-index sklearn
 pip install mne_bids
-pip install EDFlib-Python
+pip install edfio
 pip install openneuro-py
 
 # Clone down mne-iclabel and switch to the right version and install it locally

pylossless/assets/ll_default_config_adults.yaml

@@ -24,28 +24,6 @@ project:
     EEGCoordinateSystem: Other
     EEGCoordinateUnits: metres
 
-  general_info:
-    authors: [Unspecified]
-    institution_Name: Unspecified
-    institution_address: Unspecified
-    dataset_doi: []
-    funding: Unspecified
-    how_to_acknowledge: tba
-    license: ""
-    name: Unspecified
-    references_and_links: []
-
-  t_info:
-    EEG_placement_scheme: EGI 129
-    cap_manufacturer: EGI
-    cap_manufacturers_model_name: Hydrocel 129 Channel
-    hardware_filters: n/a
-    manufacturer: Electrical Geodesics
-    manufacturers_model_name: NetAmps300
-    power_line_frequency: 60
-    software_filters: n/a
-    software_versions: NetStation V4.5
-
 ######################## Task break detection ########################
 # See arguments definition from mne.preprocessing.annotate_breaks
 find_breaks:
@@ -69,19 +47,6 @@ filtering:
   notch_filter_args:
     freqs: [60]
 
-############################## SLURM #################################
-# Options for running the pipeline on a cluster
-# through SLURM
-slurm_options:
-  account: def-emayada
-  job_name: pylossless
-  memory: 16g
-  mpi: false
-  num_tasks: 1
-  program_options: []
-  threads_per_task: []
-  time_limit: 2h
-
 ########################## Nearest neighbor ##########################
 nearest_neighbors:
   n_nbr_ch: 3

pylossless/assets/ll_default_config_infants.yaml

@@ -24,28 +24,6 @@ project:
     EEGCoordinateSystem: Other
     EEGCoordinateUnits: metres
 
-  general_info:
-    authors: [Unspecified]
-    institution_Name: Unspecified
-    institution_address: Unspecified
-    dataset_doi: []
-    funding: Unspecified
-    how_to_acknowledge: tba
-    license: ""
-    name: Unspecified
-    references_and_links: []
-
-  t_info:
-    EEG_placement_scheme: EGI 129
-    cap_manufacturer: EGI
-    cap_manufacturers_model_name: Hydrocel 129 Channel
-    hardware_filters: n/a
-    manufacturer: Electrical Geodesics
-    manufacturers_model_name: NetAmps300
-    power_line_frequency: 60
-    software_filters: n/a
-    software_versions: NetStation V4.5
-
 ######################## Task break detection ########################
 # See arguments definition from mne.preprocessing.annotate_breaks
 find_breaks:
@@ -69,19 +47,6 @@ filtering:
   notch_filter_args:
     freqs: []
 
-############################## SLURM #################################
-# Options for running the pipeline on a cluster
-# through SLURM
-slurm_options:
-  account: def-emayada
-  job_name: pylossless
-  memory: 16g
-  mpi: false
-  num_tasks: 1
-  program_options: []
-  threads_per_task: []
-  time_limit: 2h
-
 ########################## Nearest neighbor ##########################
 nearest_neighbors:
   n_nbr_ch: 3

pylossless/assets/test_data/sub-s01/eeg/sub-s01_task-faceO_ll_config.yaml

@@ -92,38 +92,8 @@ project:
     coordsys:
         EEGCoordinateSystem: Other
         EEGCoordinateUnits: metres
-    general_info:
-        authors:
-        - Q1K Neuroimaging group
-        dataset_doi: []
-        funding: Azrieli Foundationt
-        how_to_acknowledge: tba
-        institution_Name: McGill University
-        institution_address: 3775 Rue University, Montreal, QC
-        license: ''
-        name: Q1K Mismatched Negativity
-        references_and_links: []
     readme: '# Q1K ACAR Dataset'
     set_montage_kwargs: {}
-    t_info:
-        EEG_placement_scheme: EGI 129
-        cap_manufacturer: EGI
-        cap_manufacturers_model_name: Hydrocel 129 Channel
-        hardware_filters: n/a
-        manufacturer: Electrical Geodesics
-        manufacturers_model_name: NetAmps300
-        power_line_frequency: 60
-        software_filters: n/a
-        software_versions: NetStation V4.5
 ref_loc_file: derivatives/EEG-IP-L/code/misc/standard_1020_ll_ref19.elc
 save_f_res: 1
-sd_t_pad: 1
-slurm_options:
-    account: def-emayada
-    job_name: pylossless
-    memory: 16g
-    mpi: false
-    num_tasks: 1
-    program_options: []
-    threads_per_task: []
-    time_limit: 2h
+sd_t_pad: 1

pylossless/assets/test_data/sub-s02/eeg/sub-s02_task-faceO_ll_config.yaml

@@ -92,38 +92,8 @@ project:
     coordsys:
         EEGCoordinateSystem: Other
         EEGCoordinateUnits: metres
-    general_info:
-        authors:
-        - Q1K Neuroimaging group
-        dataset_doi: []
-        funding: Azrieli Foundationt
-        how_to_acknowledge: tba
-        institution_Name: McGill University
-        institution_address: 3775 Rue University, Montreal, QC
-        license: ''
-        name: Q1K Mismatched Negativity
-        references_and_links: []
     readme: '# Q1K ACAR Dataset'
     set_montage_kwargs: {}
-    t_info:
-        EEG_placement_scheme: EGI 129
-        cap_manufacturer: EGI
-        cap_manufacturers_model_name: Hydrocel 129 Channel
-        hardware_filters: n/a
-        manufacturer: Electrical Geodesics
-        manufacturers_model_name: NetAmps300
-        power_line_frequency: 60
-        software_filters: n/a
-        software_versions: NetStation V4.5
 ref_loc_file: derivatives/EEG-IP-L/code/misc/standard_1020_ll_ref19.elc
 save_f_res: 1
 sd_t_pad: 1
-slurm_options:
-    account: def-emayada
-    job_name: pylossless
-    memory: 16g
-    mpi: false
-    num_tasks: 1
-    program_options: []
-    threads_per_task: []
-    time_limit: 2h

pylossless/pipeline.py

@@ -509,6 +509,7 @@ def _repr_html_(self):
             "Noisy": ch_flags.get("noisy", None),
             "Bridged": ch_flags.get("bridged", None),
             "Uncorrelated": ch_flags.get("uncorrelated", None),
+            "Rank": ch_flags.get("rank", None),
         }
         html += _create_html_details("Flagged Channels", flagged_channels_data)
 

requirements.txt

@@ -4,6 +4,8 @@ mne_bids>=0.14
 pandas
 xarray
 scipy>=1.7.1
-mne_icalabel>=0.5.0
+mne_icalabel>=0.6.0
 pyyaml
 scikit-learn
+torch
+edfio

setup.py

@@ -43,4 +43,5 @@
     extras_require=extras_require,
     include_package_data=True,
     entry_points={"console_scripts": qc_entry_point},
+    dependency_links=['https://download.pytorch.org/whl/cpu'],
 )