From fb3116d5efb2336e9084966a2ada456f86502e32 Mon Sep 17 00:00:00 2001 From: "behinger (s-ccs 001)" Date: Tue, 21 May 2024 14:09:36 +0000 Subject: [PATCH 1/8] initial new step for sync --- mne_bids_pipeline/_config.py | 7 + .../steps/preprocessing/_05b_sync_eyelink.py | 227 ++++++++++++++++++ .../steps/preprocessing/_06a1_fit_ica.py | 2 +- .../preprocessing/_06a2_find_ica_artifacts.py | 2 +- .../steps/preprocessing/_06b_run_ssp.py | 2 +- .../steps/preprocessing/_07_make_epochs.py | 2 +- .../steps/preprocessing/_08a_apply_ica.py | 2 +- .../steps/preprocessing/_08b_apply_ssp.py | 2 +- .../steps/preprocessing/__init__.py | 2 + 9 files changed, 242 insertions(+), 6 deletions(-) create mode 100644 mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py diff --git a/mne_bids_pipeline/_config.py b/mne_bids_pipeline/_config.py index 850599e36..ab3bd947f 100644 --- a/mne_bids_pipeline/_config.py +++ b/mne_bids_pipeline/_config.py @@ -1105,6 +1105,13 @@ ``` """ +sync_eyelink: bool = False + +remove_blink_saccades: bool = True +sync_eventtype_regex: str = ".*" +sync_eventtype_regex_et: str = "" + + # ### SSP, ICA, and artifact regression regress_artifact: dict[str, Any] | None = None diff --git a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py new file mode 100644 index 000000000..03f7c9447 --- /dev/null +++ b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py @@ -0,0 +1,227 @@ + +from types import SimpleNamespace + + +import mne +import re +import numpy as np +from mne_bids import BIDSPath + +from ..._config_utils import ( + _bids_kwargs, + get_eeg_reference, + get_runs, + get_sessions, + get_subjects, +) +from ..._import_data import annotations_to_events, make_epochs +from ..._logging import gen_log_kwargs, logger +from ..._parallel import get_parallel_backend, parallel_func +from ..._reject import _get_reject +from ..._report import _open_report +from ..._run import _prep_out_files, _update_for_splits, failsafe_run, save_logs + + + +def get_input_fnames_sync_eyelink( + *, + cfg: SimpleNamespace, + subject: str, + session: str | None, +) -> dict: + bids_basename = BIDSPath( + subject=subject, + session=session, + task=cfg.task, + acquisition=cfg.acq, + recording=cfg.rec, + space=cfg.space, + datatype=cfg.datatype, + root=cfg.deriv_root, + check=False, + extension=".fif", + ) + + et_bids_basename = BIDSPath( + subject=subject, + session=session, + task=cfg.task, + acquisition=cfg.acq, + recording=cfg.rec, + datatype="beh", + root=cfg.bids_root, + suffix="et", + check=False, + extension=".asc", + ) + + in_files = dict() + for run in cfg.runs: + key = f"raw_run-{run}" + in_files[key] = bids_basename.copy().update( + run=run, processing=cfg.processing, suffix="raw" + ) + _update_for_splits(in_files, key, single=True) + + + key = f"et_run-{run}" + in_files[key] = et_bids_basename.copy().update( + run=run + ) + + + + + + + return in_files + + + +@failsafe_run( + get_input_fnames=get_input_fnames_sync_eyelink, +) +def sync_eyelink( + *, + cfg: SimpleNamespace, + exec_params: SimpleNamespace, + subject: str, + session: str | None, + in_files: dict, +) -> dict: + """Run Sync for Eyelink.""" + import matplotlib.pyplot as plt + + raw_fnames = [in_files.pop(f"raw_run-{run}") for run in cfg.runs] + et_fnames = [in_files.pop(f"et_run-{run}") for run in cfg.runs] + + logger.info(**gen_log_kwargs(message=f"et_fnames {et_fnames}")) + out_files = dict() + bids_basename = raw_fnames[0].copy().update(processing=None, split=None, run=None) + out_files["eyelink"] = bids_basename.copy().update(processing="syncET", suffix="raw") + + del bids_basename + + + + for idx, (run, raw_fname,et_fname) in enumerate(zip(cfg.runs, raw_fnames,et_fnames)): + msg = f"Syncing eyelink data (fake for now) {raw_fname.basename}" + logger.info(**gen_log_kwargs(message=msg)) + raw = mne.io.read_raw_fif(raw_fname, preload=True) + raw_et = mne.io.read_raw_eyelink(et_fname) + + et_sync_times = [annotation["onset"] for annotation in raw_et.annotations if re.search(cfg.sync_eventtype_regex_et,annotation["description"])] + sync_times = [annotation["onset"] for annotation in raw.annotations if re.search(cfg.sync_eventtype_regex, annotation["description"])] + + assert len(et_sync_times) == len(sync_times),f"Detected eyetracking and EEG sync events were not of equal size ({len(et_sync_times)} vs {len(sync_times)}). Adjust your regular expressions via 'sync_eventtype_regex_et' and 'sync_eventtype_regex' accordingly" + #logger.info(**gen_log_kwargs(message=f"{et_sync_times}")) + #logger.info(**gen_log_kwargs(message=f"{sync_times}")) + + + #mne.preprocessing.eyetracking.interpolate_blinks(raw_et, buffer=(0.05, 0.05), interpolate_gaze=False) + + + # Align the data + mne.preprocessing.realign_raw(raw, raw_et, sync_times, et_sync_times) + + + # add ET data to EEG + raw.add_channels([raw_et], force_update_info=True) + + raw.set_annotations(mne.annotations._combine_annotations(raw.annotations,raw_et.annotations,0,0,0,1)) + + + + + msg = f"Saving synced data to disk." + logger.info(**gen_log_kwargs(message=msg)) + raw.save( + out_files["eyelink"], + overwrite=True, + split_size=cfg._raw_split_size, # ??? + ) + # no idea what the split stuff is... + #_update_for_splits(out_files, "epochs") + + + + # Add to report + tags = ("sync", "eyelink") + title = "Synchronize Eyelink" + with _open_report( + cfg=cfg, + exec_params=exec_params, + subject=subject, + session=session, + task=cfg.task, + ) as report: + + + caption = ( + f"The `realign_raw` function from MNE was used to align an Eyelink `asc` file to the M/EEG file." + f"The Eyelink-data was added as annotations and appended as new channels." + ) + fig = raw_et.plot(scalings=dict(eyegaze=1e3)) + report.add_figure( + fig=fig, + title="Eyelink data", + section=title, + caption=caption, + tags=tags[1], + replace=True, + ) + plt.close(fig) + del caption + return _prep_out_files(exec_params=exec_params, out_files=out_files) + + + + + + +def get_config( + *, + config: SimpleNamespace, + subject: str, + session: str | None = None, +) -> SimpleNamespace: + #logger.info(**gen_log_kwargs(message=f"config {config}")) + + cfg = SimpleNamespace( + runs=get_runs(config=config, subject=subject), + remove_blink_saccades = config.remove_blink_saccades, + sync_eventtype_regex = config.sync_eventtype_regex, + sync_eventtype_regex_et = config.sync_eventtype_regex_et, + processing= "filt" if config.regress_artifact is None else "regress", + _raw_split_size=config._raw_split_size, + + **_bids_kwargs(config=config), + ) + return cfg + + +def main(*, config: SimpleNamespace) -> None: + """Sync Eyelink.""" + if not config.sync_eyelink: + msg = "Skipping, sync_eyelink is set to False …" + logger.info(**gen_log_kwargs(message=msg, emoji="skip")) + return + + + + with get_parallel_backend(config.exec_params): + parallel, run_func = parallel_func(sync_eyelink, exec_params=config.exec_params) + logs = parallel( + run_func( + cfg=get_config(config=config, subject=subject), + exec_params=config.exec_params, + subject=subject, + session=session, + ) + for subject in get_subjects(config) + for session in get_sessions(config) + ) + save_logs(config=config, logs=logs) + + + diff --git a/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py b/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py index 598d2e308..95ff6c4a3 100644 --- a/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py +++ b/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py @@ -349,7 +349,7 @@ def get_config( eog_channels=config.eog_channels, rest_epochs_duration=config.rest_epochs_duration, rest_epochs_overlap=config.rest_epochs_overlap, - processing="filt" if config.regress_artifact is None else "regress", + processing="eyelink" if config.sync_eyelink else "filt" if config.regress_artifact is None else "regress", _epochs_split_size=config._epochs_split_size, **_bids_kwargs(config=config), ) diff --git a/mne_bids_pipeline/steps/preprocessing/_06a2_find_ica_artifacts.py b/mne_bids_pipeline/steps/preprocessing/_06a2_find_ica_artifacts.py index 43f88032a..63eff384f 100644 --- a/mne_bids_pipeline/steps/preprocessing/_06a2_find_ica_artifacts.py +++ b/mne_bids_pipeline/steps/preprocessing/_06a2_find_ica_artifacts.py @@ -365,7 +365,7 @@ def get_config( eog_channels=config.eog_channels, rest_epochs_duration=config.rest_epochs_duration, rest_epochs_overlap=config.rest_epochs_overlap, - processing="filt" if config.regress_artifact is None else "regress", + processing="eyelink" if config.sync_eyelink else "filt" if config.regress_artifact is None else "regress", **_bids_kwargs(config=config), ) return cfg diff --git a/mne_bids_pipeline/steps/preprocessing/_06b_run_ssp.py b/mne_bids_pipeline/steps/preprocessing/_06b_run_ssp.py index b17816a7e..f39b3ba98 100644 --- a/mne_bids_pipeline/steps/preprocessing/_06b_run_ssp.py +++ b/mne_bids_pipeline/steps/preprocessing/_06b_run_ssp.py @@ -249,7 +249,7 @@ def get_config( epochs_decim=config.epochs_decim, use_maxwell_filter=config.use_maxwell_filter, runs=get_runs(config=config, subject=subject), - processing="filt" if config.regress_artifact is None else "regress", + processing="eyelink" if config.sync_eyelink else "filt" if config.regress_artifact is None else "regress", **_bids_kwargs(config=config), ) return cfg diff --git a/mne_bids_pipeline/steps/preprocessing/_07_make_epochs.py b/mne_bids_pipeline/steps/preprocessing/_07_make_epochs.py index 47f717959..a52b3b0fe 100644 --- a/mne_bids_pipeline/steps/preprocessing/_07_make_epochs.py +++ b/mne_bids_pipeline/steps/preprocessing/_07_make_epochs.py @@ -335,7 +335,7 @@ def get_config( rest_epochs_overlap=config.rest_epochs_overlap, _epochs_split_size=config._epochs_split_size, runs=get_runs(config=config, subject=subject), - processing="filt" if config.regress_artifact is None else "regress", + processing="eyelink" if config.sync_eyelink else "filt" if config.regress_artifact is None else "regress", **_bids_kwargs(config=config), ) return cfg diff --git a/mne_bids_pipeline/steps/preprocessing/_08a_apply_ica.py b/mne_bids_pipeline/steps/preprocessing/_08a_apply_ica.py index 430c4cdd3..2d4d5c78a 100644 --- a/mne_bids_pipeline/steps/preprocessing/_08a_apply_ica.py +++ b/mne_bids_pipeline/steps/preprocessing/_08a_apply_ica.py @@ -243,7 +243,7 @@ def get_config( cfg = SimpleNamespace( baseline=config.baseline, ica_reject=config.ica_reject, - processing="filt" if config.regress_artifact is None else "regress", + processing="eyelink" if config.sync_eyelink else "filt" if config.regress_artifact is None else "regress", _epochs_split_size=config._epochs_split_size, **_import_data_kwargs(config=config, subject=subject), ) diff --git a/mne_bids_pipeline/steps/preprocessing/_08b_apply_ssp.py b/mne_bids_pipeline/steps/preprocessing/_08b_apply_ssp.py index 6ab00dc12..b3139433c 100644 --- a/mne_bids_pipeline/steps/preprocessing/_08b_apply_ssp.py +++ b/mne_bids_pipeline/steps/preprocessing/_08b_apply_ssp.py @@ -149,7 +149,7 @@ def get_config( subject: str, ) -> SimpleNamespace: cfg = SimpleNamespace( - processing="filt" if config.regress_artifact is None else "regress", + processing="eyelink" if config.sync_eyelink else "filt" if config.regress_artifact is None else "regress", _epochs_split_size=config._epochs_split_size, **_import_data_kwargs(config=config, subject=subject), ) diff --git a/mne_bids_pipeline/steps/preprocessing/__init__.py b/mne_bids_pipeline/steps/preprocessing/__init__.py index f9072617c..33dc66f8c 100644 --- a/mne_bids_pipeline/steps/preprocessing/__init__.py +++ b/mne_bids_pipeline/steps/preprocessing/__init__.py @@ -6,6 +6,7 @@ _03_maxfilter, _04_frequency_filter, _05_regress_artifact, + _05b_sync_eyelink, _06a1_fit_ica, _06a2_find_ica_artifacts, _06b_run_ssp, @@ -21,6 +22,7 @@ _03_maxfilter, _04_frequency_filter, _05_regress_artifact, + _05b_sync_eyelink, _06a1_fit_ica, _06a2_find_ica_artifacts, _06b_run_ssp, From ed6538f28371c96f59353455d43962a439c88909 Mon Sep 17 00:00:00 2001 From: "behinger (s-ccs 001)" Date: Tue, 21 May 2024 15:02:23 +0000 Subject: [PATCH 2/8] added extras --- mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py index 03f7c9447..5d6961e68 100644 --- a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py +++ b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py @@ -108,7 +108,7 @@ def sync_eyelink( msg = f"Syncing eyelink data (fake for now) {raw_fname.basename}" logger.info(**gen_log_kwargs(message=msg)) raw = mne.io.read_raw_fif(raw_fname, preload=True) - raw_et = mne.io.read_raw_eyelink(et_fname) + raw_et = mne.io.read_raw_eyelink(et_fname,find_overlaps=True) et_sync_times = [annotation["onset"] for annotation in raw_et.annotations if re.search(cfg.sync_eventtype_regex_et,annotation["description"])] sync_times = [annotation["onset"] for annotation in raw.annotations if re.search(cfg.sync_eventtype_regex, annotation["description"])] @@ -118,8 +118,8 @@ def sync_eyelink( #logger.info(**gen_log_kwargs(message=f"{sync_times}")) - #mne.preprocessing.eyetracking.interpolate_blinks(raw_et, buffer=(0.05, 0.05), interpolate_gaze=False) - + #mne.preprocessing.eyetracking.interpolate_blinks(raw_et, buffer=(0.05, 0.05), interpolate_gaze=True) + # Align the data mne.preprocessing.realign_raw(raw, raw_et, sync_times, et_sync_times) @@ -127,6 +127,7 @@ def sync_eyelink( # add ET data to EEG raw.add_channels([raw_et], force_update_info=True) + raw._raw_extras.append(raw_et._raw_extras) raw.set_annotations(mne.annotations._combine_annotations(raw.annotations,raw_et.annotations,0,0,0,1)) From fc7586343160ba7aa73eb07b458e64a685d6f896 Mon Sep 17 00:00:00 2001 From: jschepers Date: Thu, 23 May 2024 15:58:36 +0000 Subject: [PATCH 3/8] adapted parameters of the combine_annotations function to fix synchronization --- mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py index 5d6961e68..fadd9290c 100644 --- a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py +++ b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py @@ -129,7 +129,7 @@ def sync_eyelink( raw.add_channels([raw_et], force_update_info=True) raw._raw_extras.append(raw_et._raw_extras) - raw.set_annotations(mne.annotations._combine_annotations(raw.annotations,raw_et.annotations,0,0,0,1)) + raw.set_annotations(mne.annotations._combine_annotations(raw.annotations,raw_et.annotations,0,raw.first_samp,raw_et.first_samp,raw.info["sfreq"])) From ab6614519335711ccb18f718b1a2beccbf06739d Mon Sep 17 00:00:00 2001 From: "behinger (s-ccs 001)" Date: Tue, 28 May 2024 08:52:02 +0000 Subject: [PATCH 4/8] added edf conversion --- .../steps/preprocessing/_05b_sync_eyelink.py | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py index 5d6961e68..c87bd69a6 100644 --- a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py +++ b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py @@ -3,6 +3,7 @@ import mne +import os.path import re import numpy as np from mne_bids import BIDSPath @@ -55,6 +56,20 @@ def get_input_fnames_sync_eyelink( extension=".asc", ) + + et_edf_bids_basename = BIDSPath( + subject=subject, + session=session, + task=cfg.task, + acquisition=cfg.acq, + recording=cfg.rec, + datatype="beh", + root=cfg.bids_root, + suffix="et", + check=False, + extension=".edf", + ) + in_files = dict() for run in cfg.runs: key = f"raw_run-{run}" @@ -69,6 +84,11 @@ def get_input_fnames_sync_eyelink( run=run ) + key = f"et_edf_run-{run}" + in_files[key] = et_edf_bids_basename.copy().update( + run=run + ) + @@ -94,6 +114,7 @@ def sync_eyelink( raw_fnames = [in_files.pop(f"raw_run-{run}") for run in cfg.runs] et_fnames = [in_files.pop(f"et_run-{run}") for run in cfg.runs] + et_edf_fnames = [in_files.pop(f"et_edf_run-{run}") for run in cfg.runs] logger.info(**gen_log_kwargs(message=f"et_fnames {et_fnames}")) out_files = dict() @@ -104,10 +125,17 @@ def sync_eyelink( - for idx, (run, raw_fname,et_fname) in enumerate(zip(cfg.runs, raw_fnames,et_fnames)): + for idx, (run, raw_fname,et_fname,et_edf_fname) in enumerate(zip(cfg.runs, raw_fnames,et_fnames,et_edf_fnames)): msg = f"Syncing eyelink data (fake for now) {raw_fname.basename}" logger.info(**gen_log_kwargs(message=msg)) raw = mne.io.read_raw_fif(raw_fname, preload=True) + if not os.path.isfile(et_fname): + logger.info(**gen_log_kwargs(message=f"couldnt find {et_fname} file, trying to call edf2asc")) + if not os.path.isfile(et_edf_fname): + logger.error(**gen_log_kwargs(message=f"also didnt find {et_edf_fname} file, one of both need to exist for ET sync.")) + import subprocess + subprocess.run(["edf2asc", et_edf_fname]) + raw_et = mne.io.read_raw_eyelink(et_fname,find_overlaps=True) et_sync_times = [annotation["onset"] for annotation in raw_et.annotations if re.search(cfg.sync_eventtype_regex_et,annotation["description"])] From 1e96e2d09e6472a9643d8aa147c0bd12e68213b5 Mon Sep 17 00:00:00 2001 From: jschepers Date: Tue, 28 May 2024 16:15:40 +0000 Subject: [PATCH 5/8] Replace nan values in the ET data with zeros before synchronizing ET and EEG --- .../steps/preprocessing/_05b_sync_eyelink.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py index 8f314a369..9c92b625e 100644 --- a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py +++ b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py @@ -130,9 +130,9 @@ def sync_eyelink( logger.info(**gen_log_kwargs(message=msg)) raw = mne.io.read_raw_fif(raw_fname, preload=True) if not os.path.isfile(et_fname): - logger.info(**gen_log_kwargs(message=f"couldnt find {et_fname} file, trying to call edf2asc")) + logger.info(**gen_log_kwargs(message=f"Couldn't find {et_fname} file, trying to call edf2asc.")) if not os.path.isfile(et_edf_fname): - logger.error(**gen_log_kwargs(message=f"also didnt find {et_edf_fname} file, one of both need to exist for ET sync.")) + logger.error(**gen_log_kwargs(message=f"Also didn't find {et_edf_fname} file, one of both need to exist for ET sync.")) import subprocess subprocess.run(["edf2asc", et_edf_fname]) @@ -146,6 +146,15 @@ def sync_eyelink( #logger.info(**gen_log_kwargs(message=f"{sync_times}")) + # Check whether the eye-tracking data contains nan values. If yes replace them with zeros. + if np.isnan(raw_et.get_data()).any(): + + # Set all nan values in the eye-tracking data to 0 (to make resampling possible) + # TODO: Decide whether this is a good approch or whether interpolation (e.g. of blinks) is useful + # TODO: Decide about setting the values (e.g. for blinks) back to nan after synchronising the signals + np.nan_to_num(raw_et._data, copy=False, nan=0.0) + logger.info(**gen_log_kwargs(message=f"The eye-tracking data contained nan values. They were replaced with zeros.")) + #mne.preprocessing.eyetracking.interpolate_blinks(raw_et, buffer=(0.05, 0.05), interpolate_gaze=True) @@ -153,15 +162,14 @@ def sync_eyelink( mne.preprocessing.realign_raw(raw, raw_et, sync_times, et_sync_times) - # add ET data to EEG + # Add ET data to EEG raw.add_channels([raw_et], force_update_info=True) raw._raw_extras.append(raw_et._raw_extras) + # Also add ET annotations to EEG raw.set_annotations(mne.annotations._combine_annotations(raw.annotations,raw_et.annotations,0,raw.first_samp,raw_et.first_samp,raw.info["sfreq"])) - - msg = f"Saving synced data to disk." logger.info(**gen_log_kwargs(message=msg)) raw.save( From f54ca71841114c69fa8c92b52bba97f703e1beab Mon Sep 17 00:00:00 2001 From: jschepers Date: Wed, 29 May 2024 12:02:55 +0000 Subject: [PATCH 6/8] Use EEG sync event regex if ET sync event regex is not given --- mne_bids_pipeline/_config.py | 1 - mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mne_bids_pipeline/_config.py b/mne_bids_pipeline/_config.py index ab3bd947f..291e63e0b 100644 --- a/mne_bids_pipeline/_config.py +++ b/mne_bids_pipeline/_config.py @@ -1111,7 +1111,6 @@ sync_eventtype_regex: str = ".*" sync_eventtype_regex_et: str = "" - # ### SSP, ICA, and artifact regression regress_artifact: dict[str, Any] | None = None diff --git a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py index 9c92b625e..1ead8917f 100644 --- a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py +++ b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py @@ -137,6 +137,11 @@ def sync_eyelink( subprocess.run(["edf2asc", et_edf_fname]) raw_et = mne.io.read_raw_eyelink(et_fname,find_overlaps=True) + + # If the user did not specify a regular expression for the eye-tracking sync events, it is assumed that it's + # identical to the regex for the EEG sync events + if not cfg.sync_eventtype_regex_et: + cfg.sync_eventtype_regex_et = cfg.sync_eventtype_regex et_sync_times = [annotation["onset"] for annotation in raw_et.annotations if re.search(cfg.sync_eventtype_regex_et,annotation["description"])] sync_times = [annotation["onset"] for annotation in raw.annotations if re.search(cfg.sync_eventtype_regex, annotation["description"])] From 0c2ece14009525491f7e84399f51da7ecc829e48 Mon Sep 17 00:00:00 2001 From: jschepers Date: Tue, 11 Jun 2024 12:50:44 +0000 Subject: [PATCH 7/8] Only use EEG channels for the ICA, needs to be more general --- mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py b/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py index 95ff6c4a3..cec972d77 100644 --- a/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py +++ b/mne_bids_pipeline/steps/preprocessing/_06a1_fit_ica.py @@ -256,7 +256,8 @@ def run_ica( fit_params=fit_params, max_iter=cfg.ica_max_iterations, ) - ica.fit(epochs, decim=cfg.ica_decim) + # TODO: This works for our pipeline (exclude eye-tracking data for ICA) but probably not in general + ica.fit(epochs.pick(picks="eeg"), decim=cfg.ica_decim) explained_var = ( ica.pca_explained_variance_[: ica.n_components_].sum() / ica.pca_explained_variance_.sum() From f7e2303e3e7a6fcdb4f72676382670d0e9e616da Mon Sep 17 00:00:00 2001 From: jschepers Date: Mon, 17 Jun 2024 16:27:48 +0000 Subject: [PATCH 8/8] Added some comments and fixed mistakes --- .../steps/preprocessing/_05b_sync_eyelink.py | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py index 1ead8917f..136aecf2a 100644 --- a/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py +++ b/mne_bids_pipeline/steps/preprocessing/_05b_sync_eyelink.py @@ -1,7 +1,4 @@ - from types import SimpleNamespace - - import mne import os.path import re @@ -23,7 +20,6 @@ from ..._run import _prep_out_files, _update_for_splits, failsafe_run, save_logs - def get_input_fnames_sync_eyelink( *, cfg: SimpleNamespace, @@ -83,16 +79,13 @@ def get_input_fnames_sync_eyelink( in_files[key] = et_bids_basename.copy().update( run=run ) + _update_for_splits(in_files, key, single=True) # TODO: Find out if we need to add this or not key = f"et_edf_run-{run}" in_files[key] = et_edf_bids_basename.copy().update( run=run ) - - - - - + _update_for_splits(in_files, key, single=True) # TODO: Find out if we need to add this or not return in_files @@ -119,8 +112,7 @@ def sync_eyelink( logger.info(**gen_log_kwargs(message=f"et_fnames {et_fnames}")) out_files = dict() bids_basename = raw_fnames[0].copy().update(processing=None, split=None, run=None) - out_files["eyelink"] = bids_basename.copy().update(processing="syncET", suffix="raw") - + out_files["eyelink"] = bids_basename.copy().update(processing="eyelink", suffix="raw") del bids_basename @@ -134,7 +126,7 @@ def sync_eyelink( if not os.path.isfile(et_edf_fname): logger.error(**gen_log_kwargs(message=f"Also didn't find {et_edf_fname} file, one of both need to exist for ET sync.")) import subprocess - subprocess.run(["edf2asc", et_edf_fname]) + subprocess.run(["edf2asc", et_edf_fname]) # TODO: Still needs to be tested raw_et = mne.io.read_raw_eyelink(et_fname,find_overlaps=True) @@ -155,7 +147,7 @@ def sync_eyelink( if np.isnan(raw_et.get_data()).any(): # Set all nan values in the eye-tracking data to 0 (to make resampling possible) - # TODO: Decide whether this is a good approch or whether interpolation (e.g. of blinks) is useful + # TODO: Decide whether this is a good approach or whether interpolation (e.g. of blinks) is useful # TODO: Decide about setting the values (e.g. for blinks) back to nan after synchronising the signals np.nan_to_num(raw_et._data, copy=False, nan=0.0) logger.info(**gen_log_kwargs(message=f"The eye-tracking data contained nan values. They were replaced with zeros.")) @@ -180,10 +172,11 @@ def sync_eyelink( raw.save( out_files["eyelink"], overwrite=True, + split_naming="bids", # TODO: Find out if we need to add this or not split_size=cfg._raw_split_size, # ??? ) # no idea what the split stuff is... - #_update_for_splits(out_files, "epochs") + _update_for_splits(out_files, "eyelink") # TODO: Find out if we need to add this or not @@ -250,7 +243,6 @@ def main(*, config: SimpleNamespace) -> None: return - with get_parallel_backend(config.exec_params): parallel, run_func = parallel_func(sync_eyelink, exec_params=config.exec_params) logs = parallel(