diff --git a/README.md b/README.md index c5b0784d..1f34b47d 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,41 @@ derivatives/MEEGqc/ summary_reports/ GQI artefacts (TSV + JSON), versioned via attempt files ``` +An external output keeps the same BIDS-style structure by default: + +```bash +run-meegqc --inputdata /path/to/dataset \ + --derivatives_output /path/to/output +# /path/to/output/dataset/derivatives/MEEGqc/ +``` + +Use the opt-in literal layout when the selected folder should contain +`MEEGqc` directly: + +```bash +run-meegqc --inputdata /path/to/dataset \ + --derivatives_output /path/to/output \ + --output_layout literal +# /path/to/output/MEEGqc/ +``` + +For multiple input datasets, literal layout adds one dataset-named folder to +keep their outputs separate: `/path/to/output//MEEGqc/`. + +The Python API uses the same option: + +```python +from meg_qc.calculation.meg_qc_pipeline import make_derivative_meg_qc + +make_derivative_meg_qc( + default_config_file_path="/path/to/settings.ini", + internal_config_file_path="/path/to/settings_internal.ini", + ds_paths="/path/to/dataset", + derivatives_base="/path/to/output", + output_layout="literal", +) +``` + The GQI is written per-modality: `group_metrics/meg/Global_Quality_Index_attempt__meg.tsv` and `group_metrics/eeg/Global_Quality_Index_attempt__eeg.tsv`. diff --git a/meg_qc/calculation/meg_qc_pipeline.py b/meg_qc/calculation/meg_qc_pipeline.py index 534fcdbc..24edf540 100644 --- a/meg_qc/calculation/meg_qc_pipeline.py +++ b/meg_qc/calculation/meg_qc_pipeline.py @@ -120,9 +120,14 @@ def _safe_map_object(self, model_type, json_object, target=None): import json import pandas as pd from typing import Union, Optional, Dict, Tuple -from contextlib import contextmanager from meg_qc.calculation.metrics.summary_report_GQI import generate_gqi_summary +from meg_qc.output_paths import ( + dataset_derivatives_output, + derivative_write_context, + normalize_output_layout, + resolve_output_roots, +) # Analysis-mode names. The top-level choice is "non-profile" (write straight to @@ -159,56 +164,6 @@ def _timestamp_analysis_id() -> str: return dt.datetime.now().strftime("%Y%m%d_%H%M%S") -def resolve_output_roots(dataset_path: str, external_derivatives_root: Optional[str]) -> Tuple[str, str]: - """Return the dataset output root and derivatives folder respecting overrides. - - Parameters - ---------- - dataset_path : str - Path to the original BIDS dataset. - external_derivatives_root : Optional[str] - User-provided folder in which a dataset-named directory will be created - to host derivatives. If ``None`` the derivatives live inside the - original dataset. - - Returns - ------- - tuple - ``(output_root, derivatives_root)`` where ``output_root`` is the base - dataset directory used when writing derivatives and ``derivatives_root`` - points to the "derivatives" folder inside ``output_root``. - """ - - ds_name = os.path.basename(os.path.normpath(dataset_path)) - output_root = dataset_path if external_derivatives_root is None else os.path.join(external_derivatives_root, ds_name) - derivatives_root = os.path.join(output_root, 'derivatives') - os.makedirs(derivatives_root, exist_ok=True) - - # When output is external, seed output_root with a dataset_description.json. - # The plotting module loads ANCPBIDS directly from output_root (no symlink - # overlay) so this file must be present for schema-version detection. - if external_derivatives_root is not None: - desc_dst = os.path.join(output_root, "dataset_description.json") - if not os.path.exists(desc_dst): - desc_src = os.path.join(dataset_path, "dataset_description.json") - if os.path.exists(desc_src): - try: - import shutil as _shutil - _shutil.copy2(desc_src, desc_dst) - except OSError: - pass - if not os.path.exists(desc_dst): - import json as _json - stub = {"Name": os.path.basename(output_root), "BIDSVersion": "1.8.0"} - try: - with open(desc_dst, 'w', encoding='utf-8') as _fh: - _json.dump(stub, _fh, indent=2) - except OSError: - pass # Non-fatal; ancpbids falls back gracefully. - - return output_root, derivatives_root - - def _scan_profile_dir(profiles_root: str) -> List[Tuple[str, float]]: """Scan one profiles directory and return ``(name, mtime)`` pairs sorted by mtime desc. @@ -232,6 +187,7 @@ def _scan_profile_dir(profiles_root: str) -> List[Tuple[str, float]]: def list_analysis_profiles( dataset_path: str, external_derivatives_root: Optional[str] = None, + output_layout: str = "bids", ) -> List[str]: """List available MEGqc profile IDs for one dataset. @@ -249,7 +205,11 @@ def list_analysis_profiles( output path. Without this dual search the GUI "Load profiles" dialog shows nothing and ``resolve_analysis_root`` raises ``FileNotFoundError``. """ - _, derivatives_root = resolve_output_roots(dataset_path, external_derivatives_root) + _, derivatives_root = resolve_output_roots( + dataset_path, + external_derivatives_root, + output_layout=output_layout, + ) primary_profiles_root = os.path.join(derivatives_root, "MEEGqc", "profiles") candidates = _scan_profile_dir(primary_profiles_root) @@ -283,6 +243,7 @@ def list_analysis_profiles( def has_only_legacy_derivatives( dataset_path: str, external_derivatives_root: Optional[str] = None, + output_layout: str = "bids", ) -> bool: """Return True if old-style ('Meg_QC') derivatives exist but no new ('MEEGqc') ones. @@ -290,7 +251,11 @@ def has_only_legacy_derivatives( build reports from derivatives produced by a pre-BIDS-rename MEEGqc version. """ try: - _output_root, derivatives_root = resolve_output_roots(dataset_path, external_derivatives_root) + _output_root, derivatives_root = resolve_output_roots( + dataset_path, + external_derivatives_root, + output_layout=output_layout, + ) except Exception: return False bases = [derivatives_root] @@ -308,6 +273,7 @@ def resolve_analysis_root( analysis_mode: str = "non-profile", analysis_id: Optional[str] = None, create_if_missing: bool = False, + output_layout: str = "bids", ) -> Tuple[str, str, str, Optional[str], List[str]]: """Resolve output roots plus profile-specific MEGqc folder. @@ -325,7 +291,11 @@ def resolve_analysis_root( f"{', '.join(sorted(_ANALYSIS_MODES))}." ) - output_root, derivatives_root = resolve_output_roots(dataset_path, external_derivatives_root) + output_root, derivatives_root = resolve_output_roots( + dataset_path, + external_derivatives_root, + output_layout=output_layout, + ) non_profile_root = os.path.join(derivatives_root, "MEEGqc") if mode == "non-profile": if create_if_missing: @@ -337,7 +307,11 @@ def resolve_analysis_root( os.makedirs(profiles_root, exist_ok=True) resolved_id = analysis_id.strip() if isinstance(analysis_id, str) and analysis_id.strip() else None - available = list_analysis_profiles(dataset_path, external_derivatives_root) + available = list_analysis_profiles( + dataset_path, + external_derivatives_root, + output_layout=output_layout, + ) if mode == "new-profile": resolved_id = resolved_id or _timestamp_analysis_id() @@ -411,22 +385,6 @@ def _resolve_config_by_policy( return default_config_file_path -@contextmanager -def temporary_dataset_base(dataset, base_dir: str): - """Temporarily point an ANCPBIDS dataset to a different base directory. - - This is used to redirect derivative writing without interfering with how - raw files are located inside the original BIDS dataset. - """ - - original_base = getattr(dataset, 'base_dir_', None) - dataset.base_dir_ = base_dir - try: - yield - finally: - dataset.base_dir_ = original_base - - def _ensure_derivative_dataset_description_filename(derivative) -> None: """Guarantee a writable dataset_description filename for ANCPBIDS writes. @@ -1191,6 +1149,7 @@ def process_one_subject( derivatives_root: str, output_root: str, analysis_segments: Optional[List[str]] = None, + output_layout: str = "bids", ): """ This function processes a single subject. It contains all the code that was @@ -1213,6 +1172,8 @@ def process_one_subject( output_root : str Base directory used when persisting derivatives (parent of the derivatives folder), allowing redirection outside the BIDS dataset. + output_layout : str + Output tree convention used when persisting ANCPBIDS derivatives. analysis_segments : list of str, optional Extra folder segments inserted between ``MEEGqc`` and ``calculation``. In legacy mode this is ``[]``; in profile mode this is @@ -1684,7 +1645,12 @@ def json_writer(file_path, cont=deriv.content): print('REMOVING TRASH: FAILED') # WRITE DERIVATIVE - with temporary_dataset_base(dataset, output_root): + with derivative_write_context( + dataset, + derivative, + output_root, + output_layout, + ): ancpbids.write_derivative(dataset, derivative) # Removes intermediate trash objects — guard against NameError when no @@ -1717,7 +1683,8 @@ def process_one_subject_safe( internal_qc_params: dict, derivatives_root: str, output_root: str, - analysis_segments: Optional[List[str]] = None): + analysis_segments: Optional[List[str]] = None, + output_layout: str = "bids"): """Wrapper around :func:`process_one_subject` that catches errors. Parameters are identical to :func:`process_one_subject`. @@ -1749,6 +1716,7 @@ def process_one_subject_safe( internal_qc_params=internal_qc_params, derivatives_root=derivatives_root, output_root=output_root, + output_layout=output_layout, analysis_segments=analysis_segments, ) # process_one_subject now returns (files, metric_errors) @@ -1923,6 +1891,7 @@ def make_derivative_meg_qc( processed_subjects_policy: str = "skip", interactive_prompts: bool = False, keep_temp_on_error: bool = False, + output_layout: str = "bids", ): """Run MEGqc calculation for one or more datasets. @@ -1943,15 +1912,27 @@ def make_derivative_meg_qc( keep_temp_on_error If ``True``, temporary preprocessed FIF files are kept when an exception occurs while processing a dataset to aid debugging. + output_layout + ``"bids"`` keeps the existing external layout + ``//derivatives/MEEGqc``. ``"literal"`` writes to + ``/MEEGqc`` and requires ``derivatives_base``. """ start_time = time.time() ds_paths = check_ds_paths(ds_paths) + output_layout = normalize_output_layout(output_layout) internal_qc_params = get_internal_config_params(internal_config_file_path) requested_sub_list = sub_list + multiple_datasets = len(ds_paths) > 1 for dataset_path in ds_paths: + dataset_derivatives_base = dataset_derivatives_output( + derivatives_base, + dataset_path, + output_layout, + multiple_datasets=multiple_datasets, + ) print('___MEGqc___: ', 'DS path:', dataset_path) dataset = ancpbids.load_dataset(dataset_path, DatasetOptions(lazy_loading=True)) ( @@ -1962,7 +1943,8 @@ def make_derivative_meg_qc( analysis_segments, ) = resolve_analysis_root( dataset_path=dataset_path, - external_derivatives_root=derivatives_base, + external_derivatives_root=dataset_derivatives_base, + output_layout=output_layout, analysis_mode=analysis_mode, analysis_id=analysis_id, create_if_missing=True, @@ -2020,6 +2002,7 @@ def make_derivative_meg_qc( internal_qc_params=internal_qc_params, derivatives_root=megqc_root, output_root=output_root, + output_layout=output_layout, analysis_segments=analysis_segments, ) for sub in dataset_sub_list @@ -2080,7 +2063,12 @@ def make_derivative_meg_qc( create_config_artifact(root_folder, config_file_path, 'UsedSettings', all_subs_raw_files) # Write the pipeline-level derivative to disk - with temporary_dataset_base(dataset, output_root): + with derivative_write_context( + dataset, + derivative, + output_root, + output_layout, + ): ancpbids.write_derivative(dataset, derivative) _write_profile_manifest( diff --git a/meg_qc/output_paths.py b/meg_qc/output_paths.py new file mode 100644 index 00000000..87f87c2d --- /dev/null +++ b/meg_qc/output_paths.py @@ -0,0 +1,145 @@ +"""Output path handling shared by calculation and plotting.""" + +from __future__ import annotations + +import json +import os +import shutil +from collections.abc import Iterator +from contextlib import contextmanager + +OUTPUT_LAYOUTS = ("bids", "literal") + + +def normalize_output_layout(output_layout: str) -> str: + """Return a validated output layout name.""" + layout = str(output_layout or "bids").strip().lower() + if layout not in OUTPUT_LAYOUTS: + supported = ", ".join(OUTPUT_LAYOUTS) + raise ValueError( + f"Invalid output_layout '{output_layout}'. Supported values: {supported}." + ) + return layout + + +def dataset_derivatives_output( + derivatives_output: str | None, + dataset_path: str, + output_layout: str, + *, + multiple_datasets: bool, +) -> str | None: + """Return the external output argument to use for one dataset. + + Literal output for a multi-dataset run gets one dataset-named folder per + input. The existing BIDS layout already adds that folder in + :func:`resolve_output_roots`. + """ + layout = normalize_output_layout(output_layout) + if derivatives_output is None: + if layout == "literal": + raise ValueError("output_layout='literal' requires derivatives_output.") + return None + if layout == "literal" and multiple_datasets: + dataset_name = os.path.basename(os.path.normpath(dataset_path)) + return os.path.join(derivatives_output, dataset_name) + return derivatives_output + + +def resolve_output_roots( + dataset_path: str, + external_derivatives_root: str | None, + output_layout: str = "bids", +) -> tuple[str, str]: + """Return the output root and the folder that contains ``MEEGqc``. + + ``bids`` preserves the existing behavior. With an external output, it + writes to ``//derivatives/MEEGqc``. + + ``literal`` treats the supplied output as the derivatives folder itself, + so the result is ``/MEEGqc``. Multi-dataset callers pass a + dataset-specific output produced by :func:`dataset_derivatives_output`. + """ + layout = normalize_output_layout(output_layout) + if external_derivatives_root is None: + if layout == "literal": + raise ValueError("output_layout='literal' requires derivatives_output.") + output_root = dataset_path + derivatives_root = os.path.join(output_root, "derivatives") + elif layout == "bids": + dataset_name = os.path.basename(os.path.normpath(dataset_path)) + output_root = os.path.join(external_derivatives_root, dataset_name) + derivatives_root = os.path.join(output_root, "derivatives") + else: + output_root = external_derivatives_root + derivatives_root = output_root + + os.makedirs(derivatives_root, exist_ok=True) + + # External BIDS layout is loaded as a dataset during report generation. + # Literal layout is intentionally just an output folder and should not be + # made to look like a BIDS dataset. + if external_derivatives_root is not None and layout == "bids": + _ensure_dataset_description(output_root, dataset_path) + + return output_root, derivatives_root + + +def derivative_scope(output_layout: str, *segments: str) -> str: + """Return the ANCPBIDS query scope for the selected layout.""" + prefix = ["derivatives"] if normalize_output_layout(output_layout) == "bids" else [] + return os.path.join(*prefix, "MEEGqc", *segments) + + +@contextmanager +def derivative_write_context( + dataset, + derivative, + output_root: str, + output_layout: str, +) -> Iterator[None]: + """Temporarily point an ANCPBIDS derivative at its selected output root.""" + layout = normalize_output_layout(output_layout) + original_base = getattr(dataset, "base_dir_", None) + original_name = getattr(dataset, "name", None) + original_parent = getattr(derivative, "parent_object_", None) + + dataset.base_dir_ = output_root + if layout == "literal": + # ANCPBIDS normally places derivatives below a fixed ``derivatives`` + # parent. Reparenting only for the write makes ``MEEGqc`` land directly + # below output_root while leaving the in-memory dataset graph unchanged. + derivative.parent_object_ = dataset + dataset.name = os.path.basename(os.path.normpath(output_root)) + + try: + yield + finally: + derivative.parent_object_ = original_parent + dataset.base_dir_ = original_base + dataset.name = original_name + + +def _ensure_dataset_description(output_root: str, dataset_path: str) -> None: + """Seed an external BIDS output root with dataset metadata.""" + destination = os.path.join(output_root, "dataset_description.json") + if os.path.exists(destination): + return + + source = os.path.join(dataset_path, "dataset_description.json") + if os.path.exists(source): + try: + shutil.copy2(source, destination) + return + except OSError: + pass + + stub = { + "Name": os.path.basename(os.path.normpath(output_root)), + "BIDSVersion": "1.8.0", + } + try: + with open(destination, "w", encoding="utf-8") as file: + json.dump(stub, file, indent=2) + except OSError: + pass diff --git a/meg_qc/plotting/meg_qc_dataset_plots.py b/meg_qc/plotting/meg_qc_dataset_plots.py index d7477fa5..5e1461c5 100644 --- a/meg_qc/plotting/meg_qc_dataset_plots.py +++ b/meg_qc/plotting/meg_qc_dataset_plots.py @@ -37,6 +37,7 @@ import meg_qc from meg_qc.calculation.meg_qc_pipeline import resolve_analysis_root +from meg_qc.output_paths import resolve_output_roots as _resolve_output_roots from meg_qc.plotting.topomap_2d import make_flat_topomap_figure, BLUE_RED_COLORSCALE from meg_qc.plotting.universal_plots import amplitude_scale_unit, _add_colormap_menu_3d # Shared "Plot settings" panel (issue #136): one source of truth for the panel @@ -218,17 +219,17 @@ class ChTypeAccumulator: source_paths: set = field(default_factory=set) -def resolve_output_roots(dataset_path: str, external_derivatives_root: Optional[str]) -> Tuple[str, str]: +def resolve_output_roots( + dataset_path: str, + external_derivatives_root: Optional[str], + output_layout: str = "bids", +) -> Tuple[str, str]: """Return output root and derivatives root respecting optional override.""" - ds_name = os.path.basename(os.path.normpath(dataset_path)) - output_root = ( - dataset_path - if external_derivatives_root is None - else os.path.join(external_derivatives_root, ds_name) + return _resolve_output_roots( + dataset_path, + external_derivatives_root, + output_layout=output_layout, ) - derivatives_root = os.path.join(output_root, "derivatives") - os.makedirs(derivatives_root, exist_ok=True) - return output_root, derivatives_root def _parse_entities_from_run_key(run_key: str) -> RunMeta: @@ -7952,6 +7953,7 @@ def make_dataset_plots_meg_qc( n_jobs: int = 1, analysis_mode: str = "legacy", analysis_id: Optional[str] = None, + output_layout: str = "bids", ) -> Dict[str, Path]: """Build dataset-level QA reports from saved per-run derivatives. @@ -7970,6 +7972,9 @@ def make_dataset_plots_meg_qc( Plotting typically uses ``legacy`` or ``reuse``/``latest``. analysis_id : str, optional Profile ID used with ``analysis_mode='reuse'``. + output_layout : str + ``"bids"`` preserves the existing external tree. ``"literal"`` uses + ``derivatives_base`` as the folder containing ``MEEGqc``. Returns ------- @@ -7986,6 +7991,7 @@ def make_dataset_plots_meg_qc( ) = resolve_analysis_root( dataset_path=dataset_path, external_derivatives_root=derivatives_base, + output_layout=output_layout, analysis_mode=analysis_mode, analysis_id=analysis_id, create_if_missing=True, diff --git a/meg_qc/plotting/meg_qc_dataset_qc_plots.py b/meg_qc/plotting/meg_qc_dataset_qc_plots.py index 389d4939..ba99e308 100644 --- a/meg_qc/plotting/meg_qc_dataset_qc_plots.py +++ b/meg_qc/plotting/meg_qc_dataset_qc_plots.py @@ -39,6 +39,7 @@ import meg_qc from meg_qc.calculation.meg_qc_pipeline import resolve_analysis_root +from meg_qc.output_paths import dataset_derivatives_output METRIC_ORDER = ("GQI", "STD", "PtP", "PSD", "ECG", "EOG", "Muscle") @@ -285,6 +286,7 @@ def _resolve_input_paths( derivatives_base: Optional[str], input_tsv: Optional[str], attempt: Optional[int], + output_layout: str = "bids", analysis_mode: str = "legacy", analysis_id: Optional[str] = None, ) -> Tuple[str, str, Path, Optional[Path], Optional[int], Path]: @@ -298,6 +300,7 @@ def _resolve_input_paths( ) = resolve_analysis_root( dataset_path=dataset_path, external_derivatives_root=derivatives_base, + output_layout=output_layout, analysis_mode=analysis_mode, analysis_id=analysis_id, create_if_missing=True, @@ -365,6 +368,7 @@ def _load_one_dataset_bundle( derivatives_base: Optional[str], input_tsv: Optional[str], attempt: Optional[int], + output_layout: str = "bids", analysis_mode: str = "legacy", analysis_id: Optional[str] = None, ) -> QCDatasetBundle: @@ -380,6 +384,7 @@ def _load_one_dataset_bundle( derivatives_base=derivatives_base, input_tsv=input_tsv, attempt=attempt, + output_layout=output_layout, analysis_mode=analysis_mode, analysis_id=analysis_id, ) @@ -3379,6 +3384,7 @@ def make_dataset_qc_plots_meg_qc( derivatives_base: Optional[str] = None, analysis_mode: str = "legacy", analysis_id: Optional[str] = None, + output_layout: str = "bids", ) -> Optional[Path]: """Build one dataset-level QC HTML report from GQI summary TSV. @@ -3398,6 +3404,9 @@ def make_dataset_qc_plots_meg_qc( Analysis root selection mode (``legacy``, ``new``, ``reuse``, ``latest``). analysis_id Profile ID used with ``analysis_mode='reuse'``. + output_layout + ``"bids"`` preserves the existing external tree. ``"literal"`` uses + ``derivatives_base`` as the folder containing ``MEEGqc``. Returns ------- @@ -3410,6 +3419,7 @@ def make_dataset_qc_plots_meg_qc( derivatives_base=derivatives_base, input_tsv=input_tsv, attempt=attempt, + output_layout=output_layout, analysis_mode=analysis_mode, analysis_id=analysis_id, ) @@ -3512,6 +3522,7 @@ def make_dataset_qc_plots_multi_meg_qc( derivatives_base: Optional[str] = None, analysis_mode: str = "legacy", analysis_id: Optional[str] = None, + output_layout: str = "bids", ) -> Optional[Path]: """Build one multi-dataset QC HTML report from multiple GQI summary TSVs.""" if not dataset_paths: @@ -3520,12 +3531,19 @@ def make_dataset_qc_plots_multi_meg_qc( bundles: List[QCDatasetBundle] = [] for ds in dataset_paths: + dataset_base = dataset_derivatives_output( + derivatives_base, + ds, + output_layout, + multiple_datasets=len(dataset_paths) > 1, + ) try: bundle = _load_one_dataset_bundle( dataset_path=ds, - derivatives_base=derivatives_base, + derivatives_base=dataset_base, input_tsv=None, attempt=attempt, + output_layout=output_layout, analysis_mode=analysis_mode, analysis_id=analysis_id, ) diff --git a/meg_qc/plotting/meg_qc_multi_dataset_plots.py b/meg_qc/plotting/meg_qc_multi_dataset_plots.py index d427e239..ad97481b 100644 --- a/meg_qc/plotting/meg_qc_multi_dataset_plots.py +++ b/meg_qc/plotting/meg_qc_multi_dataset_plots.py @@ -133,6 +133,7 @@ def _collect_sample_bundle( dataset_path: str, derivatives_base: Optional[str] = None, n_jobs: int = 1, + output_layout: str = "bids", analysis_mode: str = "legacy", analysis_id: Optional[str] = None, ) -> Optional[SampleBundle]: @@ -146,6 +147,7 @@ def _collect_sample_bundle( ) = resolve_analysis_root( dataset_path=dataset_path, external_derivatives_root=derivatives_base, + output_layout=output_layout, analysis_mode=analysis_mode, analysis_id=analysis_id, create_if_missing=True, @@ -2410,6 +2412,7 @@ def make_multi_dataset_plots_meg_qc( n_jobs: int = 1, analysis_mode: str = "legacy", analysis_id: Optional[str] = None, + output_layout: str = "bids", ) -> Dict[str, Path]: """Build one HTML report comparing multiple MEGqc datasets. @@ -2429,6 +2432,9 @@ def make_multi_dataset_plots_meg_qc( Analysis root selection mode (``legacy``, ``new``, ``reuse``, ``latest``). analysis_id Profile ID used with ``analysis_mode='reuse'``. + output_layout + External output layout used to resolve every entry in + ``derivatives_bases``. Returns ------- @@ -2453,6 +2459,7 @@ def make_multi_dataset_plots_meg_qc( ds_path, der_base, n_jobs=n_jobs, + output_layout=output_layout, analysis_mode=analysis_mode, analysis_id=analysis_id, ) diff --git a/meg_qc/plotting/meg_qc_plots.py b/meg_qc/plotting/meg_qc_plots.py index fb2b310e..0f67639b 100644 --- a/meg_qc/plotting/meg_qc_plots.py +++ b/meg_qc/plotting/meg_qc_plots.py @@ -26,6 +26,11 @@ from plotly.offline import get_plotlyjs import mne from meg_qc.calculation.meg_qc_pipeline import resolve_analysis_root +from meg_qc.output_paths import ( + derivative_scope, + normalize_output_layout, + resolve_output_roots as _resolve_output_roots, +) # Get the absolute path of the parent directory of the current script parent_dir = os.path.dirname(os.getcwd()) @@ -79,6 +84,7 @@ def _load_plotting_backend(): def resolve_output_roots( dataset_path: str, external_derivatives_root: Optional[str], + output_layout: str = "bids", ) -> Tuple[str, str, str]: """Return output root plus read/write derivatives roots. @@ -87,17 +93,12 @@ def resolve_output_roots( be written (dataset root by default, external root when provided). """ - ds_name = os.path.basename(os.path.normpath(dataset_path)) - output_root = dataset_path if external_derivatives_root is None else os.path.join(external_derivatives_root, ds_name) + output_root, output_derivatives_root = _resolve_output_roots( + dataset_path, + external_derivatives_root, + output_layout=output_layout, + ) dataset_derivatives_root = os.path.join(dataset_path, 'derivatives') - output_derivatives_root = os.path.join(output_root, 'derivatives') - os.makedirs(output_derivatives_root, exist_ok=True) - - # When output is external, ensure output_root has a dataset_description.json - # so that ancpbids.load_dataset(output_root) works reliably in the plotting - # step (no symlink overlay required). - if external_derivatives_root is not None: - _ensure_output_root_bids_description(output_root, dataset_path) return output_root, dataset_derivatives_root, output_derivatives_root @@ -3608,10 +3609,14 @@ def make_plots_meg_qc( derivatives_base: Optional[str] = None, analysis_mode: str = "legacy", analysis_id: Optional[str] = None, + output_layout: str = "bids", ): """ Create plots for the MEG QC pipeline, but WITHOUT the interactive selector. Instead, we assume 'all' for every entity (subject, task, session, run, metric). + + ``output_layout='literal'`` reads and writes ``MEEGqc`` directly below + ``derivatives_base``. The default keeps the existing BIDS-style tree. """ # Ensure plotting backend and report helpers are available @@ -3628,21 +3633,22 @@ def make_plots_meg_qc( 'No data found in the given directory path! \nCheck directory path in config file and presence of data.') return + output_layout = normalize_output_layout(output_layout) ( output_root, - _derivatives_root, + output_derivatives_root, _megqc_root, _resolved_analysis_id, analysis_segments, ) = resolve_analysis_root( dataset_path=dataset_path, external_derivatives_root=derivatives_base, + output_layout=output_layout, analysis_mode=analysis_mode, analysis_id=analysis_id, create_if_missing=True, ) dataset_derivatives_root = os.path.join(dataset_path, "derivatives") - output_derivatives_root = os.path.join(output_root, "derivatives") # Query derivatives source: # - Prefer external derivatives tree when it already contains MEEGqc @@ -3668,16 +3674,24 @@ def make_plots_meg_qc( # • No os.symlink() calls → no WinError 1314 on locked-down Windows # • No temp directory or cleanup required # • Simpler code path, same result - if os.path.abspath(source_derivatives_root) != os.path.abspath(dataset_derivatives_root): - # dataset_description.json at output_root is guaranteed by resolve_output_roots - # (called earlier) but we double-check here in case plotting runs standalone. - _ensure_output_root_bids_description(output_root, dataset_path) + using_external_derivatives = ( + os.path.abspath(source_derivatives_root) + != os.path.abspath(dataset_derivatives_root) + ) + if using_external_derivatives: + # BIDS-style external roots need dataset metadata. Literal roots are + # intentionally plain output folders. + if output_layout == "bids": + _ensure_output_root_bids_description(output_root, dataset_path) query_base = output_root query_dataset = ancpbids.load_dataset(output_root, DatasetOptions(lazy_loading=True)) print(f"___MEGqc___: External output detected. Loading ANCPBIDS from output_root: {output_root}") - calculated_derivs_folder = os.path.join( - 'derivatives', 'MEEGqc', *analysis_segments, 'calculation' + source_layout = output_layout if using_external_derivatives else "bids" + calculated_derivs_folder = derivative_scope( + source_layout, + *analysis_segments, + "calculation", ) # Create output derivative folders once before subject-parallel processing. diff --git a/meg_qc/test.py b/meg_qc/test.py index 680ab451..4ac81bcd 100644 --- a/meg_qc/test.py +++ b/meg_qc/test.py @@ -6,6 +6,11 @@ import datetime as dt from typing import Callable, Dict, List, Optional, Sequence, Union +from meg_qc.output_paths import ( + dataset_derivatives_output, + normalize_output_layout, +) + def _invocation_name(fallback: str) -> str: """Return the command name the user actually typed. @@ -204,6 +209,7 @@ def run_calculation_dispatch( interactive_prompts: bool = False, keep_temp_on_error: bool = False, logger: Callable[[str], None] = print, + output_layout: str = "bids", ) -> None: """ Shared MEGqc calculation dispatcher used by CLI and GUI. @@ -216,6 +222,7 @@ def run_calculation_dispatch( ds_list = _normalize_dataset_paths(dataset_paths) if not ds_list: raise ValueError("No dataset paths provided for calculation.") + output_layout = normalize_output_layout(output_layout) analysis_mode = str(analysis_mode or "non-profile").strip().lower() if analysis_mode == "non-profile": logger( @@ -232,6 +239,12 @@ def run_calculation_dispatch( total_start = time.time() for idx, dataset_path in enumerate(ds_list, start=1): + dataset_output = dataset_derivatives_output( + derivatives_output, + dataset_path, + output_layout, + multiple_datasets=len(ds_list) > 1, + ) used_njobs = _resolve_dataset_njobs(dataset_path, n_jobs, dataset_njobs) used_subs = _resolve_dataset_sub_list(dataset_path, sub_list, dataset_subs) used_config = _resolve_dataset_config_path( @@ -252,13 +265,14 @@ def run_calculation_dispatch( ds_paths=dataset_path, sub_list=used_subs, n_jobs=used_njobs, - derivatives_base=derivatives_output, + derivatives_base=dataset_output, analysis_mode=analysis_mode, analysis_id=calc_analysis_id, existing_config_policy=existing_config_policy, processed_subjects_policy=processed_subjects_policy, interactive_prompts=interactive_prompts, keep_temp_on_error=keep_temp_on_error, + output_layout=output_layout, ) ds_elapsed = time.time() - ds_start logger( @@ -368,6 +382,7 @@ def run_plotting_dispatch( analysis_mode: str = "non-profile", analysis_id: Optional[str] = None, logger: Callable[[str], None] = print, + output_layout: str = "bids", ) -> Dict[str, bool]: """ Shared plotting dispatcher used by CLI and GUI. @@ -388,10 +403,28 @@ def run_plotting_dispatch( ) ds_list = _normalize_dataset_paths(dataset_paths) + output_layout = normalize_output_layout(output_layout) + dataset_outputs = { + ds: dataset_derivatives_output( + derivatives_output, + ds, + output_layout, + multiple_datasets=len(ds_list) > 1, + ) + for ds in ds_list + } # Refuse to build reports from derivatives produced by an older MEEGqc # version (old 'Meg_QC' folder / non-BIDS names): they are incompatible. - legacy_only = [d for d in ds_list if has_only_legacy_derivatives(d, derivatives_output)] + legacy_only = [ + d + for d in ds_list + if has_only_legacy_derivatives( + d, + dataset_outputs[d], + output_layout=output_layout, + ) + ] if legacy_only: logger("___MEGqc___: " + LEGACY_DERIVATIVES_HINT) for d in legacy_only: @@ -452,7 +485,8 @@ def run_plotting_dispatch( for ds in ds_list: _, derivatives_root, megqc_root, resolved_analysis_id, _segments = resolve_analysis_root( dataset_path=ds, - external_derivatives_root=derivatives_output, + external_derivatives_root=dataset_outputs[ds], + output_layout=output_layout, analysis_mode=effective_mode, analysis_id=effective_id, create_if_missing=True, @@ -469,9 +503,10 @@ def run_plotting_dispatch( make_plots_meg_qc( ds, n_jobs=njobs, - derivatives_base=derivatives_output, + derivatives_base=dataset_outputs[ds], analysis_mode=effective_mode, analysis_id=effective_id, + output_layout=output_layout, ) if modes["qa_dataset"]: @@ -479,15 +514,16 @@ def run_plotting_dispatch( logger(f"Running QA plotting for dataset: {ds}") make_dataset_plots_meg_qc( ds, - derivatives_base=derivatives_output, + derivatives_base=dataset_outputs[ds], n_jobs=njobs, analysis_mode=effective_mode, analysis_id=effective_id, + output_layout=output_layout, ) if modes["qa_multi_dataset"]: logger("Running QA multi-dataset plotting...") - derivatives_bases = [derivatives_output] * len(ds_list) if derivatives_output else None + derivatives_bases = [dataset_outputs[ds] for ds in ds_list] make_multi_dataset_plots_meg_qc( dataset_paths=ds_list, derivatives_bases=derivatives_bases, @@ -495,6 +531,7 @@ def run_plotting_dispatch( n_jobs=njobs, analysis_mode=effective_mode, analysis_id=effective_id, + output_layout=output_layout, ) if modes["qc_dataset"]: @@ -505,9 +542,10 @@ def run_plotting_dispatch( input_tsv=input_tsv, output_html=output_report, attempt=attempt, - derivatives_base=derivatives_output, + derivatives_base=dataset_outputs[ds_list[0]], analysis_mode=effective_mode, analysis_id=effective_id, + output_layout=output_layout, ) else: if input_tsv: @@ -521,9 +559,10 @@ def run_plotting_dispatch( input_tsv=None, output_html=None, attempt=attempt, - derivatives_base=derivatives_output, + derivatives_base=dataset_outputs[ds], analysis_mode=effective_mode, analysis_id=effective_id, + output_layout=output_layout, ) if modes["qc_multi_dataset"]: @@ -535,6 +574,7 @@ def run_plotting_dispatch( derivatives_base=derivatives_output, analysis_mode=effective_mode, analysis_id=effective_id, + output_layout=output_layout, ) return modes @@ -649,6 +689,16 @@ def run_megqc(): "Per-dataset subfolders are created automatically." ), ) + dataset_path_parser.add_argument( + "--output_layout", + choices=["bids", "literal"], + default="bids", + help=( + "External output layout. 'bids' (default) writes to " + "//derivatives/MEEGqc. 'literal' writes to " + "/MEEGqc; multi-dataset runs add one dataset folder." + ), + ) dataset_path_parser.add_argument( "--analysis_mode", type=str, @@ -774,6 +824,7 @@ def run_megqc(): calc_n_jobs=args.n_jobs, plot_njobs=args.n_jobs, derivatives_output=args.derivatives_output, + output_layout=args.output_layout, dataset_subs=subs_per_dataset, global_config_file_path=global_config_file_path, config_per_dataset=config_per_dataset, @@ -803,6 +854,7 @@ def run_megqc(): sub_list=sub_list, n_jobs=args.n_jobs, derivatives_output=args.derivatives_output, + output_layout=args.output_layout, dataset_subs=subs_per_dataset, global_config_file_path=global_config_file_path, config_per_dataset=config_per_dataset, @@ -815,8 +867,25 @@ def run_megqc(): logger=print, ) - for dataset_path in _normalize_dataset_paths(args.inputdata): - print(f"Results are available under: {dataset_path}/derivatives/MEEGqc/calculation") + from meg_qc.output_paths import resolve_output_roots + + dataset_paths = _normalize_dataset_paths(args.inputdata) + for dataset_path in dataset_paths: + dataset_output = dataset_derivatives_output( + args.derivatives_output, + dataset_path, + args.output_layout, + multiple_datasets=len(dataset_paths) > 1, + ) + _, derivatives_root = resolve_output_roots( + dataset_path, + dataset_output, + output_layout=args.output_layout, + ) + print( + "Results are available under: " + f"{os.path.join(derivatives_root, 'MEEGqc', 'calculation')}" + ) def get_config(): @@ -904,6 +973,15 @@ def get_plots(): "modes, dataset-specific derivatives are resolved from this root." ), ) + dataset_path_parser.add_argument( + "--output_layout", + choices=["bids", "literal"], + default="bids", + help=( + "External output layout. 'bids' (default) keeps a BIDS root; " + "'literal' reads and writes MEEGqc directly below the supplied path." + ), + ) dataset_path_parser.add_argument( "--analysis_mode", type=str, @@ -1003,6 +1081,7 @@ def get_plots(): run_plotting_dispatch( dataset_paths=args.inputdata, derivatives_output=args.derivatives_output, + output_layout=args.output_layout, output_report=args.output_report, attempt=args.attempt, input_tsv=args.input_tsv, @@ -1032,6 +1111,7 @@ def run_gqi_dispatch( analysis_mode: str = "non-profile", analysis_id: Optional[str] = None, logger: Callable[[str], None] = print, + output_layout: str = "bids", ) -> None: """Shared dispatcher for GQI regeneration over one or multiple datasets.""" from meg_qc.calculation.metrics.summary_report_GQI import generate_gqi_summary @@ -1044,8 +1124,26 @@ def run_gqi_dispatch( ds_list = _normalize_dataset_paths(dataset_paths) if not ds_list: raise ValueError("No dataset paths provided for GQI.") + output_layout = normalize_output_layout(output_layout) + dataset_outputs = { + ds: dataset_derivatives_output( + derivatives_output, + ds, + output_layout, + multiple_datasets=len(ds_list) > 1, + ) + for ds in ds_list + } - legacy_only = [d for d in ds_list if has_only_legacy_derivatives(d, derivatives_output)] + legacy_only = [ + d + for d in ds_list + if has_only_legacy_derivatives( + d, + dataset_outputs[d], + output_layout=output_layout, + ) + ] if legacy_only: logger("___MEGqc___: " + LEGACY_DERIVATIVES_HINT) for d in legacy_only: @@ -1074,7 +1172,8 @@ def run_gqi_dispatch( ) _, _derivatives_root, megqc_root, resolved_analysis_id, _segments = resolve_analysis_root( dataset_path=dataset_path, - external_derivatives_root=derivatives_output, + external_derivatives_root=dataset_outputs[dataset_path], + output_layout=output_layout, analysis_mode=effective_mode, analysis_id=effective_id, create_if_missing=True, @@ -1121,6 +1220,7 @@ def run_all_dispatch( qc_all: bool = False, all_modes: bool = True, logger: Callable[[str], None] = print, + output_layout: str = "bids", ) -> None: """Run MEGqc calculation (incl. GQI) and then all QA/QC plotting modes. @@ -1142,6 +1242,7 @@ def run_all_dispatch( sub_list=sub_list, n_jobs=calc_n_jobs, derivatives_output=derivatives_output, + output_layout=output_layout, dataset_njobs=dataset_njobs, dataset_subs=dataset_subs, global_config_file_path=global_config_file_path, @@ -1157,6 +1258,7 @@ def run_all_dispatch( run_plotting_dispatch( dataset_paths=dataset_paths, derivatives_output=derivatives_output, + output_layout=output_layout, njobs=plot_njobs, qa_subject=qa_subject, qa_dataset=qa_dataset, @@ -1206,6 +1308,15 @@ def run_gqi(): required=False, help="Optional folder to store derivatives outside the BIDS dataset", ) + parser.add_argument( + "--output_layout", + choices=["bids", "literal"], + default="bids", + help=( + "External output layout. 'bids' (default) writes a BIDS derivatives " + "tree; 'literal' uses the supplied path directly." + ), + ) parser.add_argument( "--analysis_mode", type=str, @@ -1236,6 +1347,7 @@ def run_gqi(): dataset_paths=args.inputdata, default_config_file_path=default_config, derivatives_output=args.derivatives_output, + output_layout=args.output_layout, global_config_file_path=args.config, config_per_dataset=config_per_dataset, analysis_mode=args.analysis_mode, diff --git a/tests/realdata/test_literal_output_layout.py b/tests/realdata/test_literal_output_layout.py new file mode 100644 index 00000000..d37f2e4a --- /dev/null +++ b/tests/realdata/test_literal_output_layout.py @@ -0,0 +1,40 @@ +"""End-to-end coverage for literal external output.""" + +import pytest + +pytestmark = pytest.mark.realdata + + +def test_literal_output_calculation_and_plotting( + one_meg, + isolated_dataset, + fast_config, + cli, + tmp_path, +): + dataset = isolated_dataset(one_meg[1]) + output = tmp_path / "quality-control" + common = [ + "--inputdata", + str(dataset), + "--derivatives_output", + str(output), + "--output_layout", + "literal", + ] + + calculation = cli( + ["run-meegqc", *common, "--config", str(fast_config), "--n_jobs", "1"] + ) + assert calculation.returncode == 0, calculation.stdout[-3000:] + + calculation_root = output / "MEEGqc" / "calculation" + assert calculation_root.is_dir() + assert list(calculation_root.rglob("*_desc-STDs_*.tsv")) + assert not (output / dataset.name).exists() + assert not (output / "derivatives").exists() + + plotting = cli(["run-meegqc-plotting", *common, "--qa-subject"]) + assert plotting.returncode == 0, plotting.stdout[-3000:] + reports = output / "MEEGqc" / "reports" + assert list(reports.glob("*/sub-*/*subjectQaReport*.html")) diff --git a/tests/test_output_layout.py b/tests/test_output_layout.py new file mode 100644 index 00000000..2f8986cc --- /dev/null +++ b/tests/test_output_layout.py @@ -0,0 +1,150 @@ +"""Tests for external MEEGqc output layouts.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import ancpbids +import pytest + +from meg_qc.output_paths import ( + dataset_derivatives_output, + derivative_scope, + derivative_write_context, + resolve_output_roots, +) + + +def _make_dataset(path: Path, name: str = "source") -> Path: + path.mkdir() + (path / "dataset_description.json").write_text( + json.dumps({"Name": name, "BIDSVersion": "1.8.0"}), + encoding="utf-8", + ) + return path + + +def _write_test_derivative(dataset_path: Path, output_root: Path, layout: str) -> None: + dataset = ancpbids.load_dataset(str(dataset_path)) + derivative = dataset.create_derivative(name="MEEGqc") + derivative.dataset_description.GeneratedBy.Name = "MEEGqc" + + folder = derivative.create_folder(name="calculation") + artifact = folder.create_artifact() + artifact.add_entity("desc", "layout") + artifact.suffix = "test" + artifact.extension = ".txt" + artifact.content = lambda file_path: Path(file_path).write_text( + "ok", encoding="utf-8" + ) + + original_base = dataset.base_dir_ + original_name = dataset.name + original_parent = derivative.parent_object_ + with derivative_write_context(dataset, derivative, str(output_root), layout): + ancpbids.write_derivative(dataset, derivative) + + assert dataset.base_dir_ == original_base + assert dataset.name == original_name + assert derivative.parent_object_ is original_parent + + +def test_default_output_stays_inside_dataset(tmp_path): + dataset_path = _make_dataset(tmp_path / "dataset") + + output_root, derivatives_root = resolve_output_roots( + str(dataset_path), None, output_layout="bids" + ) + + assert Path(output_root) == dataset_path + assert Path(derivatives_root) == dataset_path / "derivatives" + + _write_test_derivative(dataset_path, Path(output_root), "bids") + expected = dataset_path / "derivatives" / "MEEGqc" / "calculation" + assert list(expected.glob("*_test.txt")) + + +def test_external_bids_layout_keeps_existing_tree(tmp_path): + dataset_path = _make_dataset(tmp_path / "dataset", name="original") + external_root = tmp_path / "external" + + output_root, derivatives_root = resolve_output_roots( + str(dataset_path), str(external_root), output_layout="bids" + ) + + assert Path(output_root) == external_root / dataset_path.name + assert Path(derivatives_root) == Path(output_root) / "derivatives" + copied = json.loads( + (Path(output_root) / "dataset_description.json").read_text(encoding="utf-8") + ) + assert copied["Name"] == "original" + + _write_test_derivative(dataset_path, Path(output_root), "bids") + expected = Path(derivatives_root) / "MEEGqc" / "calculation" + assert list(expected.glob("*_test.txt")) + + +def test_literal_layout_uses_requested_folder(tmp_path): + dataset_path = _make_dataset(tmp_path / "dataset") + literal_root = tmp_path / "chosen-output" + + output_root, derivatives_root = resolve_output_roots( + str(dataset_path), str(literal_root), output_layout="literal" + ) + + assert Path(output_root) == literal_root + assert Path(derivatives_root) == literal_root + assert not (literal_root / "dataset_description.json").exists() + + _write_test_derivative(dataset_path, literal_root, "literal") + expected = literal_root / "MEEGqc" / "calculation" + assert list(expected.glob("*_test.txt")) + assert not (literal_root / "derivatives").exists() + + loaded = ancpbids.load_dataset(str(literal_root)) + files = loaded.query( + scope=derivative_scope("literal", "calculation"), + return_type="filename", + ) + assert len(files) == 1 + assert Path(files[0]).name == "desc-layout_test.txt" + + +def test_literal_multi_dataset_outputs_are_separate(tmp_path): + output_root = tmp_path / "external" + first = _make_dataset(tmp_path / "dataset-one", name="first") + second = _make_dataset(tmp_path / "dataset-two", name="second") + + first_output = dataset_derivatives_output( + str(output_root), str(first), "literal", multiple_datasets=True + ) + second_output = dataset_derivatives_output( + str(output_root), str(second), "literal", multiple_datasets=True + ) + + assert Path(first_output) == output_root / "dataset-one" + assert Path(second_output) == output_root / "dataset-two" + assert first_output != second_output + + for dataset_path, dataset_output in ( + (first, first_output), + (second, second_output), + ): + _, derivatives_root = resolve_output_roots( + str(dataset_path), dataset_output, output_layout="literal" + ) + _write_test_derivative(dataset_path, Path(derivatives_root), "literal") + assert list( + (Path(dataset_output) / "MEEGqc" / "calculation").glob("*_test.txt") + ) + + +def test_literal_layout_requires_external_output(tmp_path): + with pytest.raises(ValueError, match="requires derivatives_output"): + resolve_output_roots(str(tmp_path / "dataset"), None, "literal") + + +def test_unknown_output_layout_is_rejected(tmp_path): + with pytest.raises(ValueError, match="Invalid output_layout"): + resolve_output_roots(str(tmp_path / "dataset"), str(tmp_path), "flat")