MAINT: Refactor _get_raw_paths (mne-tools#749)

Author: larsoner

docs/source/settings/general.md

@@ -28,6 +28,7 @@
         - eeg_template_montage
         - drop_channels
         - reader_extra_params
+        - read_raw_bids_verbose
         - analyze_channels
         - plot_psd_for_runs
         - n_jobs

docs/source/v1.4.md.inc

@@ -3,19 +3,20 @@
 ### :new: New features & enhancements
 
 - Add movement compensation and cHPI filtering to the Maxwell filtering step, along with additional configuration options (#747 by @larsoner)
-- Add option to specify `ssp_ecg_channel` to override the default value (#747 by @larsoner)
+- Add option to specify [`ssp_ecg_channel`]([mne_bids_pipeline._config.ssp_ecg_channel) to override the default value (#747 by @larsoner)
+- Add option [`read_raw_bids_verbose`]([mne_bids_pipeline._config.read_raw_bids_verbose) to set the verbosity level when using `read_raw_bids` to suppress known warnings (#749 by @larsoner)
 
 [//]: # (### :warning: Behavior changes)
 
 [//]: # (- Whatever (#000 by @whoever))
 
 ### :medical_symbol: Code health
 
-- Refactor code to deduplicate keyword-passing (#746 by @larsoner)
+- Refactor code to deduplicate keyword-passing and improve internal logic (#746, #749 by @larsoner)
 
 ### :bug: Bug fixes
 
-- Fix bug when `mf_reference_run != runs[0]` (#742 by @larsoner)
+- Fix bug when [`mf_reference_run != runs[0]`]([mne_bids_pipeline._config.mf_reference_run) (#742 by @larsoner)
 - Fix bug with too many JSON files found during empty room matching (#743 by @allermat)
 - Fix bug with outdated info on ch_types config option (#745 by @allermat)
 - Fix bug where SSP projectors were not added to the report (#747 by @larsoner)

mne_bids_pipeline/_config.py

@@ -416,6 +416,14 @@
     ```
 """
 
+read_raw_bids_verbose: Optional[Literal["error"]] = None
+"""
+Verbosity level to pass to `read_raw_bids(..., verbose=read_raw_bids_verbose)`.
+If you know your dataset will contain files that are not perfectly BIDS
+compliant (e.g., "Did not find any meg.json..."), you can set this to
+`'error'` to suppress warnings emitted by read_raw_bids.
+"""
+
 ###############################################################################
 # BREAK DETECTION
 # ---------------

mne_bids_pipeline/_config_utils.py

@@ -118,16 +118,6 @@ def get_sessions(config: SimpleNamespace) -> Union[List[None], List[str]]:
         return sessions
 
 
-@functools.lru_cache(maxsize=None)
-def _get_runs_all_subjects_cached(
-    **config_dict: Dict[str, Any],
-) -> Dict[str, Union[List[None], List[str]]]:
-    config = SimpleNamespace(**config_dict)
-    # Sometimes we check list equivalence for ch_types, so convert it back
-    config.ch_types = list(config.ch_types)
-    return get_runs_all_subjects(config)
-
-
 def get_runs_all_subjects(
     config: SimpleNamespace,
 ) -> Dict[str, Union[List[None], List[str]]]:
@@ -139,7 +129,26 @@ def get_runs_all_subjects(
     for each subject asked in the configuration file
     (and not for each subject present in the bids_path).
     """
-    # We cannot use get_subjects() because if there is just one subject
+    # Use caching under the hood for speed
+    return copy.deepcopy(
+        _get_runs_all_subjects_cached(
+            bids_root=config.bids_root,
+            data_type=config.data_type,
+            ch_types=tuple(config.ch_types),
+            subjects=tuple(config.subjects) if config.subjects != "all" else "all",
+            exclude_subjects=tuple(config.exclude_subjects),
+            exclude_runs=tuple(config.exclude_runs) if config.exclude_runs else None,
+        )
+    )
+
+
+@functools.lru_cache(maxsize=None)
+def _get_runs_all_subjects_cached(
+    **config_dict: Dict[str, Any],
+) -> Dict[str, Union[List[None], List[str]]]:
+    config = SimpleNamespace(**config_dict)
+    # Sometimes we check list equivalence for ch_types, so convert it back
+    config.ch_types = list(config.ch_types)
     subj_runs = dict()
     for subject in get_subjects(config):
         # Only traverse through the current subject's directory
@@ -193,15 +202,7 @@ def get_runs(
         return [None]
 
     runs = copy.deepcopy(config.runs)
-
-    subj_runs = _get_runs_all_subjects_cached(
-        bids_root=config.bids_root,
-        data_type=config.data_type,
-        ch_types=tuple(config.ch_types),
-        subjects=tuple(config.subjects) if config.subjects != "all" else "all",
-        exclude_subjects=tuple(config.exclude_subjects),
-        exclude_runs=tuple(config.exclude_runs) if config.exclude_runs else None,
-    )
+    subj_runs = get_runs_all_subjects(config=config)
     valid_runs = subj_runs[subject]
 
     if len(get_subjects(config)) > 1:
@@ -239,24 +240,28 @@ def get_runs_tasks(
     config: SimpleNamespace,
     subject: str,
     session: Optional[str],
+    include_noise: bool = True,
 ) -> List[Tuple[str]]:
     """Get (run, task) tuples for all runs plus (maybe) rest."""
-    from ._import_data import _get_bids_path_in
+    from ._import_data import _get_noise_path, _get_rest_path
 
     runs = get_runs(config=config, subject=subject)
-    tasks = [config.task] * len(runs)
-    if config.process_rest and not config.task_is_rest:
-        run, task = None, "rest"
-        bids_path_in = _get_bids_path_in(
-            cfg=config,
-            subject=subject,
-            session=session,
-            run=run,
-            task=task,
-        )
-        if bids_path_in.fpath.exists():
+    tasks = [get_task(config=config)] * len(runs)
+    kwargs = dict(
+        cfg=config,
+        subject=subject,
+        session=session,
+        kind="orig",
+        add_bads=False,
+    )
+    if _get_rest_path(**kwargs):
+        runs.append(None)
+        tasks.append("rest")
+    if include_noise:
+        mf_reference_run = get_mf_reference_run(config=config)
+        if _get_noise_path(mf_reference_run=mf_reference_run, **kwargs):
             runs.append(None)
-            tasks.append("rest")
+            tasks.append("noise")
     return tuple(zip(runs, tasks))
 
 
@@ -600,3 +605,7 @@ def _bids_kwargs(*, config: SimpleNamespace) -> dict:
         bids_root=config.bids_root,
         deriv_root=config.deriv_root,
     )
+
+
+def _do_mf_autobad(*, cfg: SimpleNamespace) -> bool:
+    return cfg.find_noisy_channels_meg or cfg.find_flat_channels_meg