[REF] UKB : Modify tsv writing to fit BIDS specifications (aramis-lab#1526)

* Split write-bids function

* Add write function specific to images

* WIP 1206

* get_ext_from_sidecars

* write_scans

* Unit test for participants

* Add test for sessions

* Add test for scans

* Add UKB to nonregression

Author: AliceJoubert

clinica/converters/ukb_to_bids/_utils.py

@@ -3,6 +3,11 @@
 
 import numpy as np
 import pandas as pd
+from fsspec.implementations.local import LocalFileSystem
+
+from clinica.converters._utils import write_to_tsv
+from clinica.dataset.bids._filename import Extension
+from clinica.utils.stream import cprint
 
 __all__ = [
     "find_clinical_data",
@@ -303,17 +308,21 @@ def write_bids(
     scans: pd.DataFrame,
     dataset_directory: Path,
 ) -> None:
-    from fsspec.implementations.local import LocalFileSystem
+    fs = LocalFileSystem(auto_mkdir=True)
+    _write_description_and_participants(participants, to, fs)
+    _write_sessions(sessions, to, fs)
+    _write_images(scans, to, dataset_directory, fs)
+    _write_scans(scans, to)
 
-    from clinica.converters._utils import write_to_tsv
+
+def _write_description_and_participants(
+    participants: pd.DataFrame, to: Path, fs: LocalFileSystem
+):
     from clinica.converters.study_models import StudyName
     from clinica.dataset import BIDSDatasetDescription
 
-    fs = LocalFileSystem(auto_mkdir=True)
-
-    participants = participants.droplevel(
-        ["sessions", "modality", "bids_filename"]
-    ).drop_duplicates()
+    participants = participants.droplevel(["sessions", "modality", "bids_filename"])
+    participants = participants.loc[~participants.index.duplicated(keep="first")]
 
     # Ensure BIDS hierarchy is written first.
     with fs.transaction:
@@ -326,72 +335,79 @@ def write_bids(
         with fs.open(str(to / "participants.tsv"), "w") as participant_file:
             write_to_tsv(participants, participant_file)
 
+
+def _write_sessions(sessions: pd.DataFrame, to: Path, fs: LocalFileSystem):
     for participant_id, data_frame in sessions.groupby("participant_id"):
-        sessions = data_frame.droplevel(
+        sessions_to_write = data_frame.droplevel(
             ["participant_id", "modality", "bids_filename"]
         ).drop_duplicates()
-
+        sessions_to_write.index.name = "session_id"
         sessions_filepath = to / str(participant_id) / f"{participant_id}_sessions.tsv"
         with fs.open(str(sessions_filepath), "w") as sessions_file:
-            write_to_tsv(sessions, sessions_file)
+            write_to_tsv(sessions_to_write, sessions_file)
 
-    scans = scans.reset_index().set_index(["bids_full_path"], verify_integrity=True)
 
+def _write_images(scans: pd.DataFrame, to: Path, source: Path, fs: LocalFileSystem):
+    scans = scans.reset_index().set_index(["bids_full_path"], verify_integrity=True)
     for bids_full_path, metadata in scans.iterrows():
         if metadata["modality_num"] != "20217" and metadata["modality_num"] != "20225":
             _copy_file_to_bids(
-                zipfile=dataset_directory / metadata["source_zipfile"],
+                zipfile=source / metadata["source_zipfile"],
                 filenames=[metadata["source_filename"]] + metadata["sidecars"],
                 bids_path=to / bids_full_path,
             )
         else:
             _convert_dicom_to_nifti(
-                zipfiles=dataset_directory / metadata["source_zipfile"],
+                zipfiles=source / metadata["source_zipfile"],
                 bids_path=to / bids_full_path,
+                fs=fs,
             )
             if metadata["modality_num"] == "20217":
-                _import_event_tsv(bids_path=to)
-
-        _write_row_in_scans_tsv_file(metadata, to)
-
-
-def _write_row_in_scans_tsv_file(row: pd.Series, to: Path):
-    """Write rows from a dataframe into a scans.tsv file.
+                _import_event_tsv(bids_path=to, fs=fs)
 
-    Parameters
-    ----------
-    row : pd.Series
-        Row to write into the scans.tsv file.
-
-    to : Path
-        Path to the BIDS folder.
-    """
-    scans_filepath = (
-        to
-        / str(row.participant_id)
-        / str(row.sessions)
-        / f"{row.participant_id}_{row.sessions}_scans.tsv"
-    )
-    row_to_write = _serialize_row(
-        row.drop(["participant_id", "sessions"]),
-        write_column_names=not scans_filepath.exists(),
-    )
-    with open(scans_filepath, "a") as scans_file:
-        scans_file.write(f"{row_to_write}\n")
 
+def _get_extensions_from_sidecars(sidecars: list[str]) -> list[str]:
+    extensions = []
+    for side in sidecars:
+        try:
+            extensions += [Extension("." + side.split(".")[1])]
+        except (ValueError, IndexError) as e:
+            cprint(
+                "An invalid extension for bids files was found and won't be registered in scans.tsv. Please check your files.",
+                lvl="warning",
+            )
+    return extensions + [Extension(".nii.gz")]
 
-def _serialize_row(row: pd.Series, write_column_names: bool) -> str:
-    row_dict = row.to_dict()
-    to_write = (
-        [row_dict.keys(), row_dict.values()]
-        if write_column_names
-        else [row_dict.values()]
-    )
-    return "\n".join([_serialize_list(list(_)) for _ in to_write])
 
+def _write_scans(scans: pd.DataFrame, to: Path) -> None:
+    for subject_session, data in scans.groupby(["participant_id", "sessions"]):
+        data["filename_no_extension"] = data["bids_full_path"].apply(
+            lambda x: f"{Path(x).parent.name}/{Path(x).name}"
+        )
+        data["extensions"] = data["sidecars"].apply(
+            lambda x: _get_extensions_from_sidecars(x)
+        )
 
-def _serialize_list(data: list, sep="\t") -> str:
-    return sep.join([str(value) for value in data])
+        to_write = pd.DataFrame(columns=["filename"])
+
+        for _, line in data.iterrows():
+            for extension in line.extensions:
+                to_write = pd.concat(
+                    [
+                        to_write,
+                        pd.DataFrame(
+                            {"filename": [line.filename_no_extension + extension]}
+                        ),
+                    ]
+                )
+        to_write.to_csv(
+            to
+            / subject_session[0]
+            / subject_session[1]
+            / f"{subject_session[0]}_{subject_session[1]}_scans.tsv",
+            sep="\t",
+            index=False,
+        )
 
 
 def _copy_file_to_bids(zipfile: Path, filenames: List[Path], bids_path: Path) -> None:
@@ -408,7 +424,9 @@ def _copy_file_to_bids(zipfile: Path, filenames: List[Path], bids_path: Path) ->
                 f.write(fs.cat(filename))
 
 
-def _convert_dicom_to_nifti(zipfiles: Path, bids_path: Path) -> None:
+def _convert_dicom_to_nifti(
+    zipfiles: Path, bids_path: Path, fs: LocalFileSystem
+) -> None:
     """Install the requested files in the BIDS  dataset.
     First, the dicom is extracted in a temporary directory
     Second, the dicom extracted is converted in the right place using dcm2niix"""
@@ -418,10 +436,6 @@ def _convert_dicom_to_nifti(zipfiles: Path, bids_path: Path) -> None:
     import zipfile
     from pathlib import PurePath
 
-    from fsspec.implementations.local import LocalFileSystem
-
-    fs = LocalFileSystem(auto_mkdir=True)
-
     zf = zipfile.ZipFile(zipfiles)
     try:
         bids_path.parent.mkdir(exist_ok=True, parents=True)
@@ -430,15 +444,9 @@ def _convert_dicom_to_nifti(zipfiles: Path, bids_path: Path) -> None:
         pass
     with tempfile.TemporaryDirectory() as tempdir:
         zf.extractall(tempdir)
-        command = [
-            "dcm2niix",
-            "-w",
-            "0",
-        ]
-        command += ["-9", "-z", "y"]
-        command += ["-b", "y", "-ba", "y"]
-        command += [tempdir]
-        subprocess.run(command)
+        subprocess.run(
+            ["dcm2niix", "-w", "0", "-9", "-z", "y", "-b", "y", "-ba", "y", tempdir]
+        )
         fmri_image_path = _find_largest_imaging_data(Path(tempdir))
         fmri_image_path = fmri_image_path or ""
         fs.copy(str(fmri_image_path), str(bids_path) + ".nii.gz")
@@ -485,11 +493,8 @@ def _select_sessions(x: pd.Series) -> Optional[float]:
     return None
 
 
-def _import_event_tsv(bids_path: Path) -> None:
+def _import_event_tsv(bids_path: Path, fs: LocalFileSystem) -> None:
     """Import the csv containing the events' information."""
-    from fsspec.implementations.local import LocalFileSystem
-
-    fs = LocalFileSystem(auto_mkdir=True)
     event_tsv = (
         Path(__file__).parents[2]
         / "resources"

test/nonregression/test_run_converters.py

@@ -42,5 +42,6 @@ def test_converters(cmdopt, tmp_path, study: StudyName):
         StudyName.NIFD,
         StudyName.OASIS3,
         StudyName.GENFI,
+        StudyName.UKB,
     ):
         compare_bids_tsv(output_dir, ref_dir / "bids")

test/unittests/converters/test_ukb_to_bids_utils.py

@@ -3,6 +3,7 @@
 
 import pandas as pd
 import pytest
+from fsspec.implementations.local import LocalFileSystem
 
 
 def test_read_imaging_data(tmp_path):
@@ -20,33 +21,20 @@ def test_read_imaging_data(tmp_path):
         read_imaging_data(path_to_zip)
 
 
-def test_write_row_in_scans_tsv_file(tmp_path):
-    from clinica.converters.ukb_to_bids._utils import _write_row_in_scans_tsv_file
-
-    row = pd.Series(
-        {
-            "participant_id": "sub-0001",
-            "sessions": "ses-M000",
-            "filename": "sub-0001_ses-M000_T1w.nii.gz",
-            "modality": "T1w",
-        }
-    )
-
-    target_dir = tmp_path / "BIDS" / "sub-0001" / "ses-M000"
-    target_dir.mkdir(parents=True)
-
-    _write_row_in_scans_tsv_file(row, tmp_path / "BIDS")
+@pytest.mark.parametrize(
+    "sidecars, expected", [([], {".nii.gz"}), (["truc.json"], {".nii.gz", ".json"})]
+)
+def test_get_extensions_from_sidecars_success(sidecars, expected):
+    from clinica.converters.ukb_to_bids._utils import _get_extensions_from_sidecars
 
-    scans_tsv = target_dir / "sub-0001_ses-M000_scans.tsv"
-    assert scans_tsv.exists()
+    assert expected == set(_get_extensions_from_sidecars(sidecars))
 
-    content = scans_tsv.read_text().strip().splitlines()
 
-    columns_names = content[0].split("\t")
-    columns_items = content[1].split("\t")
+@pytest.mark.parametrize("sidecars", [["foo"], [".bar"]])
+def test_get_extensions_from_sidecars_error(sidecars):
+    from clinica.converters.ukb_to_bids._utils import _get_extensions_from_sidecars
 
-    assert columns_names == ["filename", "modality"]
-    assert columns_items == ["sub-0001_ses-M000_T1w.nii.gz", "T1w"]
+    assert _get_extensions_from_sidecars(sidecars) == [".nii.gz"]
 
 
 @pytest.mark.parametrize(
@@ -72,3 +60,89 @@ def test_select_sessions(subject_id, source_session, age_2, age_3, expected):
     )
 
     assert expected == _select_sessions(clinical_data)
+
+
+def test_write_description_and_participants(tmp_path):
+    from clinica.converters.ukb_to_bids._utils import (
+        _write_description_and_participants,
+    )
+
+    to = tmp_path / "BIDS"
+    participants = pd.DataFrame(
+        {
+            "participants": ["1", "2", "2"],
+            "sessions": ["ses-M000", "ses-M000", "ses-M001"],
+            "modality": ["dwi", "dwi", "dwi"],
+            "bids_filename": ["1-0-dwi", "2-0-dwi", "2-1-dwi"],
+            "sex": ["F", "F", "F"],
+        }
+    )
+    participants.set_index(
+        ["participants", "sessions", "modality", "bids_filename"], inplace=True
+    )
+    _write_description_and_participants(
+        participants, to, LocalFileSystem(auto_mkdir=True)
+    )
+
+    tsv_files = list(to.rglob("*tsv"))
+    json_files = list(to.rglob("*json"))
+
+    assert len(tsv_files) == 1
+    assert len(json_files) == 1
+
+    tsv = pd.read_csv(tsv_files[0], sep="\t")
+    assert set(tsv.columns) == {"participants", "sex"}
+    assert len(tsv) == 2
+
+
+def test_write_sessions(tmp_path):
+    from clinica.converters.ukb_to_bids._utils import _write_sessions
+
+    to = tmp_path / "BIDS"
+
+    sessions = pd.DataFrame(
+        {
+            "participant_id": ["1", "2", "2"],
+            "sessions": ["ses-M000", "ses-M000", "ses-M001"],
+            "modality": ["dwi", "dwi", "dwi"],
+            "bids_filename": ["1-0-dwi", "2-0-dwi", "2-1-dwi"],
+            "session_identifier": ["2", "2", "3"],
+        }
+    )
+    sessions.set_index(
+        ["participant_id", "sessions", "modality", "bids_filename"], inplace=True
+    )
+
+    _write_sessions(sessions, to, LocalFileSystem(auto_mkdir=True))
+    tsv_files = list(to.rglob("*tsv"))
+
+    assert len(tsv_files) == 2
+
+    tsv = pd.read_csv(to / "2" / "2_sessions.tsv", sep="\t")
+    assert len(tsv) == 2
+
+
+def test_write_scans(tmp_path):
+    from clinica.converters.ukb_to_bids._utils import _write_scans
+
+    to = tmp_path / "BIDS"
+    (to / "sub-001" / "ses-M000").mkdir(parents=True, exist_ok=True)
+    scans = pd.DataFrame(
+        pd.DataFrame(
+            {
+                "participant_id": ["sub-001"],
+                "sessions": ["ses-M000"],
+                "modality": ["T1w"],
+                "bids_filename": ["sub-001_ses-M000_T1w"],
+                "bids_full_path": [
+                    to / "sub-001" / "ses-M000" / "sub-001_ses-M000_T1w"
+                ],
+                "sidecars": [["truc.json"]],
+            }
+        )
+    )
+    _write_scans(scans, to)
+    tsv_files = list(to.rglob("*tsv"))
+    assert len(tsv_files) == 1
+    tsv = pd.read_csv(tsv_files[0], sep="\t")
+    assert len(tsv) == 2