Minor tweaks

Author: marcelzwiers

bidscoin/bids.py

@@ -985,9 +985,9 @@ def load_bidsmap(yamlfile: Path=Path(), folder: Path=templatefolder, plugins:Ite
     for dataformat in bidsmap:
         if dataformat in ('$schema', 'Options'): continue
         bidsmap[dataformat]['session'] = bidsmap[dataformat]['session'] or ''   # Session-less data repositories
-        for datatype in bidsmap[dataformat]:
-            if not isinstance(bidsmap[dataformat][datatype], list): continue    # E.g. 'subject', 'session' and empty datatypes
-            for index, run in enumerate(bidsmap[dataformat][datatype]):
+        for datatype in bidsmap[dataformat] or []:
+            if datatype in ('subject', 'session'): continue
+            for index, run in enumerate(bidsmap[dataformat][datatype] or []):
 
                 # Add missing provenance info
                 if not run.get('provenance'):
@@ -1045,12 +1045,11 @@ def save_bidsmap(filename: Path, bidsmap: Bidsmap) -> None:
     :return:
     """
 
-    # Remove the added DataSource object
+    # Remove the added DataSource objects
     bidsmap = copy.deepcopy(bidsmap)
     for dataformat in bidsmap:
         if dataformat in ('$schema', 'Options'): continue
-        if not bidsmap[dataformat]: continue
-        for datatype in bidsmap[dataformat]:
+        for datatype in bidsmap[dataformat] or []:
             if not isinstance(bidsmap[dataformat][datatype], list): continue    # E.g. 'subject' and 'session'
             for run in bidsmap[dataformat][datatype]:
                 run.pop('datasource', None)
@@ -1091,8 +1090,7 @@ def validate_bidsmap(bidsmap: Bidsmap, level: int=1) -> bool:
     LOGGER.info(f"bids-validator {bids_validator.__version__} test results (* = in .bidsignore):")
     for dataformat in bidsmap:
         if dataformat in ('$schema', 'Options'): continue
-        if not bidsmap[dataformat]: continue
-        for datatype in bidsmap[dataformat]:
+        for datatype in bidsmap[dataformat] or []:
             if not isinstance(bidsmap[dataformat][datatype], list): continue        # E.g. 'subject' and 'session'
             for run in bidsmap[dataformat][datatype]:
                 bidsname = get_bidsname(f"sub-{sanitize(dataformat)}", '', run, False)
@@ -1135,8 +1133,7 @@ def check_bidsmap(bidsmap: Bidsmap, checks: Tuple[bool, bool, bool]=(True, True,
     LOGGER.info('Checking the bidsmap run-items:')
     for dataformat in bidsmap:
         if dataformat in ('$schema', 'Options'): continue    # TODO: Check Options
-        if not bidsmap[dataformat]: continue
-        for datatype in bidsmap[dataformat]:
+        for datatype in bidsmap[dataformat] or []:
             if not isinstance(bidsmap[dataformat][datatype], list):                  continue   # E.g. 'subject' and 'session'
             if datatype in bidsmap['Options']['bidscoin']['ignoretypes']:            continue   # E.g. 'exclude'
             if check_ignore(datatype, bidsmap['Options']['bidscoin']['bidsignore']): continue
@@ -1178,7 +1175,7 @@ def check_template(bidsmap: Bidsmap) -> bool:
     LOGGER.verbose('Checking the template bidsmap datatypes:')
     for dataformat in bidsmap:
         if dataformat in ('$schema', 'Options'): continue
-        for datatype in bidsmap[dataformat]:
+        for datatype in bidsmap[dataformat] or []:
             if not isinstance(bidsmap[dataformat][datatype], list): continue        # Skip datatype = 'subject'/'session'
             if not (datatype in bidsdatatypesdef or datatype in ignoretypes or check_ignore(datatype, bidsignore)):
                 LOGGER.warning(f"Invalid {dataformat} datatype: '{datatype}' (you may want to add it to the 'bidsignore' list)")
@@ -1427,7 +1424,7 @@ def create_run(datasource: DataSource=None, bidsmap: Bidsmap=None) -> Run:
     return Run(dict(provenance = str(datasource.path),
                     properties = {'filepath':'', 'filename':'', 'filesize':'', 'nrfiles':None},
                     attributes = {},
-                    bids       = {},
+                    bids       = {'suffix':''},
                     meta       = {},
                     datasource = datasource))
 
@@ -1518,21 +1515,21 @@ def find_run(bidsmap: Bidsmap, provenance: str, dataformat: str='', datatype: st
     return Run({})
 
 
-def delete_run(bidsmap: Bidsmap, provenance: Union[Run, str], datatype: str= '', dataformat: str='') -> None:
+def delete_run(bidsmap: Bidsmap, provenance: Union[Run, str], datatype: str= '', dataformat: str='') -> bool:
     """
     Delete the first matching run from the BIDS map
 
     :param bidsmap:     Full bidsmap data structure, with all options, BIDS labels and attributes, etc.
     :param provenance:  The provenance identifier of/or the run-item that is deleted
     :param datatype:    The datatype that of the deleted run_item (can be different from run_item['datasource']), e.g. 'anat'
     :param dataformat:  The dataformat section in the bidsmap in which the run is deleted, e.g. 'DICOM'
-    :return:
+    :return:            True if successful, False otherwise
     """
 
     if isinstance(provenance, str):
         run_item = find_run(bidsmap, provenance, dataformat)
         if not run_item:
-            return
+            return False
     else:
         run_item   = provenance
         provenance = run_item['provenance']
@@ -1545,17 +1542,19 @@ def delete_run(bidsmap: Bidsmap, provenance: Union[Run, str], datatype: str= '',
         for index, run in enumerate(bidsmap[dataformat].get(datatype,[])):
             if Path(run['provenance']) == Path(provenance):
                 del bidsmap[dataformat][datatype][index]
-                return
+                return True
 
     LOGGER.error(f"Could not find (and delete) this [{dataformat}][{datatype}] run: '{provenance}")
+    return False
 
 
-def append_run(bidsmap: Bidsmap, run: Run) -> None:
+def insert_run(bidsmap: Bidsmap, run: Run, position: int=None) -> None:
     """
-    Append a cleaned-up run to the BIDS map
+    Inserts a cleaned-up run to the BIDS map
 
     :param bidsmap:     Full bidsmap data structure, with all options, BIDS labels and attributes, etc.
     :param run:         The run (listitem) that is appended to the datatype
+    :param position:    The position at which the run is inserted. The run is appended at the end if position is None
     :return:
     """
 
@@ -1574,7 +1573,7 @@ def append_run(bidsmap: Bidsmap, run: Run) -> None:
     if not bidsmap.get(dataformat).get(datatype):
         bidsmap[dataformat][datatype] = [run]
     else:
-        bidsmap[dataformat][datatype].append(run)
+        bidsmap[dataformat][datatype].insert(len(bidsmap[dataformat][datatype]) if position is None else position, run)
 
 
 def update_bidsmap(bidsmap: Bidsmap, source_datatype: str, run: Run) -> None:
@@ -1610,7 +1609,7 @@ def update_bidsmap(bidsmap: Bidsmap, source_datatype: str, run: Run) -> None:
         delete_run(bidsmap, run, source_datatype)
 
         # Append the (cleaned-up) target run
-        append_run(bidsmap, run)
+        insert_run(bidsmap, run)
 
     else:
         for index, run_ in enumerate(bidsmap[dataformat][run_datatype]):

bidscoin/bidscoiner.py

@@ -270,11 +270,11 @@ def bidscoiner(sourcefolder: str, bidsfolder: str, participant: list=(), force:
                     subid, sesid = datasource.subid_sesid(subid, sesid or '')
                     bidssession  = bidsfolder/subid/sesid       # TODO: Support DICOMDIR with multiple subjects (as in PYDICOMDIR)
                     if not force and bidssession.is_dir():
-                        datatypes = []
+                        datatypes = set()
                         for dataformat in dataformats:
                             for datatype in lsdirs(bidssession):                               # See what datatypes we already have in the bids session-folder
                                 if list(datatype.iterdir()) and bidsmap[dataformat].get(datatype.name): # See if we are going to add data for this datatype
-                                    datatypes.append(datatype.name)
+                                    datatypes.add(datatype.name)
                         if datatypes:
                             LOGGER.info(f">>> Skipping processed session: {bidssession} already has {datatypes} data (you can carefully use the -f option to overrule)")
                             continue

bidscoin/bidsmapper.py

@@ -91,7 +91,7 @@ def bidsmapper(sourcefolder: str, bidsfolder: str, bidsmap: str, template: str,
         unzip = bidsmap_new['Options']['bidscoin'].get('unzip','')
     for dataformat in bidsmap_new:
         if dataformat in ('$schema', 'Options'): continue
-        for datatype in bidsmap_new[dataformat]:
+        for datatype in bidsmap_new[dataformat] or []:
             if datatype not in ('subject', 'session'):
                 bidsmap_new[dataformat][datatype] = []
 
@@ -248,7 +248,7 @@ def main():
 
     trackusage('bidsmapper')
     try:
-        bidsmapper(**args(args))
+        bidsmapper(**vars(args))
 
     except Exception:
         trackusage('bidsmapper_exception')

bidscoin/heuristics/schema.json

@@ -1,6 +1,6 @@
 {
    "$schema": "https://json-schema.org/draft/2020-12/schema",
-   "$id": "http://bidscoin.templates/schema.json",
+   "$id": "file://bidscoin/heuristics/schema.json",
    "title": "Schema for validating template bidsmaps",
    "type": "object",
    "required": ["Options"],
@@ -53,8 +53,14 @@
       "required": ["subject", "session"],
 
       "properties": {
-         "subject": { "type": "string" },
-         "session": { "type": ["string", "integer", "null"] }
+         "subject": {
+            "type": "string",
+            "description": "The participant label (typically a dynamic value)"
+         },
+         "session": {
+            "type": ["string", "integer", "null"],
+            "description": "The session label (typically a dynamic value)"
+         }
       },
 
       "additionalProperties": {
@@ -63,7 +69,7 @@
          "items": {
             "description": "Run-item (containing the bidsmappings)",
             "type": "object",
-            "required": ["bids"],
+            "required": ["provenance", "bids"],
             "properties": {
                "provenance": {
                   "description": "The fullpath name of the data source. Serves also as a unique identifier to find a run-item in the bidsmap",

bidscoin/plugins/dcm2niix2bids.py

@@ -196,7 +196,7 @@ def bidsmapper_plugin(session: Path, bidsmap_new: Bidsmap, bidsmap_old: Bidsmap,
                 run['datasource'].path = targetfile
 
             # Copy the filled-in run over to the new bidsmap
-            bids.append_run(bidsmap_new, run)
+            bids.insert_run(bidsmap_new, run)
 
         else:
             LOGGER.bcdebug(f"Existing/duplicate '{datasource.datatype}' {dataformat} sample: {sourcefile}")
@@ -372,7 +372,7 @@ def bidscoiner_plugin(session: Path, bidsmap: Bidsmap, bidsses: Path) -> Union[N
             dcm2niixpostfixes = ('_c', '_i', '_Eq', '_real', '_imaginary', '_MoCo', '_t', '_Tilt', '_e', '_ph', '_ADC', '_fieldmaphz')              #_c%d, _e%d and _ph (and any combination of these in that order) are for multi-coil data, multi-echo data and phase data
             dcm2niixfiles = sorted(set([dcm2niixfile for dcm2niixpostfix in dcm2niixpostfixes for dcm2niixfile in outfolder.glob(f"{bidsname}*{dcm2niixpostfix}*.nii*")]))
             dcm2niixfiles = [dcm2niixfile for dcm2niixfile in dcm2niixfiles if not (re.match(r'sub-.*_echo-[0-9]*\.nii',      dcm2niixfile.name) or
-                                                                                    re.match(r'sub-.*_phase(diff|[12])\.nii', dcm2niixfile.name))]   # Skip false-positive (-> glob) dcm2niixfiles, e.g. postfix = 'echo-1' (see Github issue #232)
+                                                                                    re.match(r'sub-.*_phase(diff|[12])\.nii', dcm2niixfile.name))]   # Skip false-positive (-> glob) dcm2niixfiles, e.g. postfix = 'echo-1' (see GitHub issue #232)
 
             # Rename all dcm2niix files that got additional postfixes (i.e. store the postfixes in the bidsname)
             for dcm2niixfile in dcm2niixfiles:

bidscoin/plugins/nibabel2bids.py

@@ -152,7 +152,7 @@ def bidsmapper_plugin(session: Path, bidsmap_new: Bidsmap, bidsmap_old: Bidsmap,
                 run['datasource'].path = targetfile
 
             # Copy the filled-in run over to the new bidsmap
-            bids.append_run(bidsmap_new, run)
+            bids.insert_run(bidsmap_new, run)
 
         else:
             LOGGER.bcdebug(f"Existing/duplicate '{datasource.datatype}' {datasource.dataformat} sample: {sourcefile}")

bidscoin/plugins/spec2nii2bids.py

@@ -157,7 +157,7 @@ def bidsmapper_plugin(session: Path, bidsmap_new: Bidsmap, bidsmap_old: Bidsmap,
                 run['datasource'].path = targetfile
 
             # Copy the filled-in run over to the new bidsmap
-            bids.append_run(bidsmap_new, run)
+            bids.insert_run(bidsmap_new, run)
 
         else:
             LOGGER.bcdebug(f"Existing/duplicate '{datasource.datatype}' {dataformat} sample: {sourcefile}")

docs/preparation.rst

@@ -4,7 +4,10 @@ Data organization
 Supported source data structures
 --------------------------------
 
-Out of the box, BIDScoin requires that the source data repository is organized according to a ``subject/[session]/data`` structure (the ``session`` subfolder is always optional). The ``data`` folder(s) can be structured in various ways (depending on the plugin and/or dataformat), as illustrated by the following examples:
+Out of the box, BIDScoin requires that the source data repository is organized according to a ``subject/[session]/data`` structure (the ``session`` subfolder is always optional). The ``data`` folder(s) can be structured in various ways (depending on the plugin and/or dataformat), as illustrated by in the sections below.
+
+.. note::
+   You can store your session data as zipped (``.zip``) or tarzipped (e.g. ``.tar.gz``) archive files. BIDScoin `workflow tools <./workflow.html>`__ will automatically unpack/unzip those archive files in a temporary folder and then process your session data from there. For flat/DICOMDIR data, BIDScoin tools (i.e. the bidsmapper and the bidscoiner) will automatically run `dicomsort <./utilities.html#dicomsort>`__ in a temporary folder to sort them in seriesfolders. Depending on the data and file system, repeatedly unzipping data in the workflow may come with a significant processing speed penalty. BIDScoin plugins will skip (Linux-style hidden) files and folders of which the name starts with a ``.`` (dot) character.
 
 1. A DICOM Series layout
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -142,12 +145,6 @@ The above layouts are supported by the (default) `dcm2niix2bids <./plugins.html#
     |   [..]
     [..]
 
-.. note::
-   You can store your session data in any of the above data layouts as zipped (``.zip``) or tarzipped (e.g. ``.tar.gz``) archive files. BIDScoin `workflow tools <./workflow.html>`__ will automatically unpack/unzip those archive files in a temporary folder and then process your session data from there. For flat/DICOMDIR data, BIDScoin tools (i.e. the bidsmapper and the bidscoiner) will automatically run `dicomsort <./utilities.html#dicomsort>`__ in a temporary folder to sort them in seriesfolders. Depending on the data and file system, repeatedly unzipping data in the workflow may come with a significant processing speed penalty.
-
-.. tip::
-   BIDScoin plugins will typically skip (Linux-style hidden) files and folders of which the name starts with a ``.`` (dot) character. You can use this feature to flexibly omit subjects, sessions or runs from your bids repository, for instance when you restarted an MRI scan because something went wrong with the stimulus presentation and you don't want that data to be converted and enumerated as ``run-1``, ``run-2``.
-
 Recommended data acquisition conventions
 ----------------------------------------
 

docs/troubleshooting.rst

@@ -3,7 +3,7 @@ Troubleshooting
 
 Installation
 ------------
-A first step when encountering execution errors is to test whether your installation is working correctly. An easy way to test the working of various BIDScoin components is to run ``bidscoin -t`` in your terminal. Some commonly seen messages are:
+When encountering execution errors it is helpful to know whether your BIDScoin installation is working correctly. An easy way to test the working of its various components is to run ``bidscoin -t`` in your terminal. Some commonly reported issues are:
 
 The "dcm2niix" command is not recognized
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -41,7 +41,7 @@ The fix comes from these resources:
 
 Workflow
 --------
-The first step in troubleshooting is to look at the warnings and messages printed out in the terminal (they are also save to disk in the ``bidsfolder/code/bidscoin`` output folder). Make sure you are OK with the warnings (they are meaningful and not to be ignored) and do not continue with a next step until all errors are resolved.
+The first step in workflow troubleshooting is to look at the warnings and messages printed out in the terminal (they are also saved to disk in the ``bidsfolder/code/bidscoin`` output folder). Make sure you are OK with the warnings (they are meaningful and not to be ignored) and do not continue with a next step until all errors are resolved. Below you can find answers to some commonly reported issues.
 
 My bidsmap is empty
 ^^^^^^^^^^^^^^^^^^^
@@ -105,6 +105,10 @@ My source-files can no longer be found
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 You may get the warning "Cannot reliably change the data type and/or suffix because the source file '..' can no longer be found". This warning is generated when (1) your source data moved to a different location, or (2) your data is zipped or in DICOMDIR format. This warning can be ignored if you do not need to change the data type of your run-items anymore (in the bidseditor), because in that case BIDScoin may need access to the source data (to read new properties or attributes). To restore data access for (1), move the data to it's original location and for (2) use the ``--store`` option of bidsmapper to store local copies of the source data samples in the bids output folder.
 
+Some acquisitions went wrong and need to be excluded
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+BIDScoin plugins will skip (Linux-style hidden) files and folders of which the name starts with a ``.`` (dot) character. You can use this feature to flexibly omit subjects, sessions or runs from your bids repository, for instance when you restarted an MRI scan because something went wrong with the stimulus presentation and you don't want that data to be converted and enumerated as ``run-1``, ``run-2``.
+
 I have duplicated field maps because of an interrupted session
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 It may happen that due to irregularities during data acquisition you had to reacquire your field-map for part of your data. In that case the ``IntendedFor`` and ``B0FieldIdentifier``/``B0FieldSource`` semantics become ambiguous. To handle this situation, you can use json sidecar files to extend the source attributes (see below) or use the limited ``IntendedFor`` search as described `here <./bidsmap.html#intendedfor>`__ and `here <https://github.com/Donders-Institute/bidscoin/issues/123>`__.
@@ -136,4 +140,4 @@ You can simply use the ``bidseditor`` to make changes to your bidsmap, delete al
 
 More help
 ---------
-If this guide does not help to solve your problem, then you can `search on github <https://github.com/Donders-Institute/bidscoin/issues?q=>`__ for open and/or closed issues to see if anyone else has encountered similar problems before. If not, feel free to help yourself and others by opening a new github issue.
+If this guide does not help to solve your problem, then you can `search on github <https://github.com/Donders-Institute/bidscoin/issues?q=>`__ for open and/or closed issues to see if anyone else has encountered similar problems before. If not, feel free to help yourself and others by opening a new github issue or post your question on `NeuroStars <https://neurostars.org/tag/bidscoin>`__ (tag: #bidscoin)