Merge pull request #2194 from FCP-INDI/many_pipes

🐛 Bugfixes for many pipelines project

Author: sgiavasis

.circleci/main.yml

@@ -45,10 +45,6 @@ commands:
       - run:
           name: "Configuring git user"
           command: |
-            sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 78BD65473CB3BD13
-            curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
-            sudo apt-get update
-            sudo apt-get install git -y
             git config --global user.email "[email protected]"
             git config --global user.name "Theodore (machine user) @ CircleCI"
   create-docker-test-container:
@@ -64,11 +60,6 @@ commands:
             mkdir -p ~/project/test-results
             docker pull ${DOCKER_TAG}
             docker run -v /etc/passwd:/etc/passwd --user=$(id -u):c-pac -dit -P -e COVERAGE_FILE=<< parameters.coverage-file >> -v /home/circleci/project/test-results:/code/test-results -v /home/circleci:/home/circleci -v /home/circleci/project/CPAC/resources/configs/test_configs:/test_configs -v $PWD:/code -v $PWD/dev/circleci_data:$PWD/dev/circleci_data --workdir=/home/circleci/project --entrypoint=/bin/bash --name docker_test ${DOCKER_TAG}
-  get-sample-bids-data:
-    steps:
-      - run:
-          name: Getting Sample BIDS Data
-          command: git clone https://github.com/bids-standard/bids-examples.git
   get-singularity:
     parameters:
       version:
@@ -231,7 +222,6 @@ jobs:
       - set-up-variant:
           variant: "<< parameters.variant >>"
       - set-python-version
-      - get-sample-bids-data
       - run-pytest-docker
       - store_test_results:
           path: test-results

CHANGELOG.md

@@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Required positional parameter "wf" in input and output of `ingress_pipeconfig_paths` function, where a node to reorient templates is added to the `wf`.
 - Required positional parameter "orientation" to `resolve_resolution`.
 - Optional positional argument "cfg" to `create_lesion_preproc`.
+- Allow enabling `overwrite_transform` only when the registration method is `ANTS`.
+- `resource_inventory` utility to inventory NodeBlock function inputs and outputs.
 - New switch `mask_sbref` under `func_input_prep` in functional registration and set to default `on`.
 - New resource `desc-head_bold` as non skull-stripped bold from nodeblock `bold_masking`.
 - `censor_file_path` from `offending_timepoints_connector` in the `build_nuisance_regressor` node.
@@ -33,6 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Moved `pygraphviz` from requirements to `graphviz` optional dependencies group.
 - Automatically tag untagged `subject_id` and `unique_id` as `!!str` when loading data config files.
 - Made orientation configurable (was hard-coded as "RPI").
+- Resource-not-found errors now include information about where to source those resources.
 - Moved `ref_mask_res_2` and `T1w_template_res-2` fields from registration into surface under `abcd_prefreesurfer_prep`.
 - Moved `find_censors node` inside `create_nuisance_regression_workflow` into its own function/subworkflow as `offending_timepoints_connector`.
 - [FSL-AFNI subworkflow](https://github.com/FCP-INDI/C-PAC/blob/4bdd6c410ef0a9b90f53100ea005af1f7d6e76c0/CPAC/func_preproc/func_preproc.py#L1052C4-L1231C25)

CPAC/_entrypoints/run.py

@@ -454,6 +454,14 @@ def run_main():
         action="store_true",
     )
 
+    parser.add_argument(
+        "--freesurfer_dir",
+        "--freesurfer-dir",
+        help="Specify path to pre-computed FreeSurfer outputs "
+        "to pull into C-PAC run",
+        default=False,
+    )
+
     # get the command line arguments
     args = parser.parse_args(
         sys.argv[1 : (sys.argv.index("--") if "--" in sys.argv else len(sys.argv))]
@@ -743,6 +751,9 @@ def run_main():
             c["pipeline_setup", "system_config", "num_participants_at_once"],
         )
 
+        if args.freesurfer_dir:
+            c["pipeline_setup"]["freesurfer_dir"] = args.freesurfer_dir
+
         if not args.data_config_file:
             WFLOGGER.info("Input directory: %s", bids_dir)
 
@@ -783,9 +794,8 @@ def run_main():
             sub_list = load_cpac_data_config(
                 args.data_config_file, args.participant_label, args.aws_input_creds
             )
-        list(sub_list)
         sub_list = sub_list_filter_by_labels(
-            sub_list, {"T1w": args.T1w_label, "bold": args.bold_label}
+            list(sub_list), {"T1w": args.T1w_label, "bold": args.bold_label}
         )
 
         # C-PAC only handles single anatomical images (for now)

CPAC/_global_fixtures.py

@@ -0,0 +1,34 @@
+# Copyright (C) 2025  C-PAC Developers
+
+# This file is part of C-PAC.
+
+# C-PAC is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+
+# C-PAC is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+# License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
+"""Global fixtures for C-PAC tests."""
+
+from pathlib import Path
+
+from _pytest.tmpdir import TempPathFactory
+from git import Repo
+import pytest
+
+
+@pytest.fixture(scope="session")
+def bids_examples(tmp_path_factory: TempPathFactory) -> Path:
+    """Get the BIDS examples dataset."""
+    example_dir = tmp_path_factory.mktemp("bids-examples")
+    if not example_dir.exists() or not any(example_dir.iterdir()):
+        Repo.clone_from(
+            "https://github.com/bids-standard/bids-examples.git", str(example_dir)
+        )
+    return example_dir

CPAC/anat_preproc/anat_preproc.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2012-2023  C-PAC Developers
+# Copyright (C) 2012-2025  C-PAC Developers
 
 # This file is part of C-PAC.
 
@@ -2572,7 +2572,7 @@ def brain_mask_acpc_niworkflows_ants_T2(wf, cfg, strat_pool, pipe_num, opt=None)
     config=["anatomical_preproc", "brain_extraction"],
     option_key="using",
     option_val="UNet",
-    inputs=["desc-preproc_T2w", "T1w-brain-template", "T1w-template", "unet_model"],
+    inputs=["desc-preproc_T2w", "T1w-brain-template", "T1w-template", "unet-model"],
     outputs=["space-T2w_desc-brain_mask"],
 )
 def brain_mask_unet_T2(wf, cfg, strat_pool, pipe_num, opt=None):
@@ -2586,7 +2586,7 @@ def brain_mask_unet_T2(wf, cfg, strat_pool, pipe_num, opt=None):
     config=["anatomical_preproc", "brain_extraction"],
     option_key="using",
     option_val="UNet",
-    inputs=["desc-preproc_T2w", "T1w-brain-template", "T1w-template", "unet_model"],
+    inputs=["desc-preproc_T2w", "T1w-brain-template", "T1w-template", "unet-model"],
     outputs=["space-T2w_desc-acpcbrain_mask"],
 )
 def brain_mask_acpc_unet_T2(wf, cfg, strat_pool, pipe_num, opt=None):

CPAC/conftest.py

@@ -0,0 +1,21 @@
+# Copyright (C) 2025  C-PAC Developers
+
+# This file is part of C-PAC.
+
+# C-PAC is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+
+# C-PAC is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+# License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
+"""Global fixtures for C-PAC tests."""
+
+from CPAC._global_fixtures import bids_examples
+
+__all__ = ["bids_examples"]

CPAC/func_preproc/func_preproc.py

@@ -1503,6 +1503,91 @@ def bold_mask_anatomical_based(wf, cfg, strat_pool, pipe_num, opt=None):
     return (wf, outputs)
 
 
+def anat_brain_to_bold_res(wf_name, cfg, pipe_num):
+    wf = pe.Workflow(name=f"{wf_name}_{pipe_num}")
+
+    inputNode = pe.Node(
+        util.IdentityInterface(
+            fields=["T1w-template-funcreg", "space-template_desc-preproc_T1w"]
+        ),
+        name="inputspec",
+    )
+    outputNode = pe.Node(
+        util.IdentityInterface(fields=["space-template_res-bold_desc-brain_T1w"]),
+        name="outputspec",
+    )
+
+    # applywarp --rel --interp=spline -i ${T1wImage} -r ${ResampRefIm} --premat=$FSLDIR/etc/flirtsch/ident.mat -o ${WD}/${T1wImageFile}.${FinalfMRIResolution}
+    anat_brain_to_func_res = pe.Node(
+        interface=fsl.ApplyWarp(), name=f"resample_anat_brain_in_standard_{pipe_num}"
+    )
+
+    anat_brain_to_func_res.inputs.interp = "spline"
+    anat_brain_to_func_res.inputs.premat = cfg.registration_workflows[
+        "anatomical_registration"
+    ]["registration"]["FSL-FNIRT"]["identity_matrix"]
+
+    wf.connect(
+        inputNode, "space-template_desc-preproc_T1w", anat_brain_to_func_res, "in_file"
+    )
+    wf.connect(inputNode, "T1w-template-funcreg", anat_brain_to_func_res, "ref_file")
+
+    wf.connect(
+        anat_brain_to_func_res,
+        "out_file",
+        outputNode,
+        "space-template_res-bold_desc-brain_T1w",
+    )
+    return wf
+
+
+def anat_brain_mask_to_bold_res(wf_name, cfg, pipe_num):
+    # Create brain masks in this space from the FreeSurfer output (changing resolution)
+    # applywarp --rel --interp=nn -i ${FreeSurferBrainMask}.nii.gz -r ${WD}/${T1wImageFile}.${FinalfMRIResolution} --premat=$FSLDIR/etc/flirtsch/ident.mat -o ${WD}/${FreeSurferBrainMaskFile}.${FinalfMRIResolution}.nii.gz
+    wf = pe.Workflow(name=f"{wf_name}_{pipe_num}")
+    inputNode = pe.Node(
+        util.IdentityInterface(
+            fields=["space-template_desc-brain_mask", "space-template_desc-preproc_T1w"]
+        ),
+        name="inputspec",
+    )
+    outputNode = pe.Node(
+        util.IdentityInterface(fields=["space-template_desc-bold_mask"]),
+        name="outputspec",
+    )
+
+    anat_brain_mask_to_func_res = pe.Node(
+        interface=fsl.ApplyWarp(),
+        name=f"resample_anat_brain_mask_in_standard_{pipe_num}",
+    )
+
+    anat_brain_mask_to_func_res.inputs.interp = "nn"
+    anat_brain_mask_to_func_res.inputs.premat = cfg.registration_workflows[
+        "anatomical_registration"
+    ]["registration"]["FSL-FNIRT"]["identity_matrix"]
+
+    wf.connect(
+        inputNode,
+        "space-template_desc-brain_mask",
+        anat_brain_mask_to_func_res,
+        "in_file",
+    )
+    wf.connect(
+        inputNode,
+        "space-template_desc-preproc_T1w",
+        anat_brain_mask_to_func_res,
+        "ref_file",
+    )
+    wf.connect(
+        anat_brain_mask_to_func_res,
+        "out_file",
+        outputNode,
+        "space-template_desc-bold_mask",
+    )
+
+    return wf
+
+
 @nodeblock(
     name="bold_mask_anatomical_resampled",
     switch=[
@@ -1528,39 +1613,35 @@ def bold_mask_anatomical_resampled(wf, cfg, strat_pool, pipe_num, opt=None):
 
     Adapted from `DCAN Lab's BOLD mask method from the ABCD pipeline <https://github.com/DCAN-Labs/DCAN-HCP/blob/1d90814/fMRIVolume/scripts/OneStepResampling.sh#L121-L132>`_.
     """
-    # applywarp --rel --interp=spline -i ${T1wImage} -r ${ResampRefIm} --premat=$FSLDIR/etc/flirtsch/ident.mat -o ${WD}/${T1wImageFile}.${FinalfMRIResolution}
-    anat_brain_to_func_res = pe.Node(
-        interface=fsl.ApplyWarp(), name=f"resample_anat_brain_in_standard_{pipe_num}"
-    )
-
-    anat_brain_to_func_res.inputs.interp = "spline"
-    anat_brain_to_func_res.inputs.premat = cfg.registration_workflows[
-        "anatomical_registration"
-    ]["registration"]["FSL-FNIRT"]["identity_matrix"]
+    anat_brain_to_func_res = anat_brain_to_bold_res(wf, cfg, pipe_num)
 
     node, out = strat_pool.get_data("space-template_desc-preproc_T1w")
-    wf.connect(node, out, anat_brain_to_func_res, "in_file")
+    wf.connect(
+        node, out, anat_brain_to_func_res, "inputspec.space-template_desc-preproc_T1w"
+    )
 
     node, out = strat_pool.get_data("T1w-template-funcreg")
-    wf.connect(node, out, anat_brain_to_func_res, "ref_file")
+    wf.connect(node, out, anat_brain_to_func_res, "inputspec.T1w-template-funcreg")
 
     # Create brain masks in this space from the FreeSurfer output (changing resolution)
     # applywarp --rel --interp=nn -i ${FreeSurferBrainMask}.nii.gz -r ${WD}/${T1wImageFile}.${FinalfMRIResolution} --premat=$FSLDIR/etc/flirtsch/ident.mat -o ${WD}/${FreeSurferBrainMaskFile}.${FinalfMRIResolution}.nii.gz
-    anat_brain_mask_to_func_res = pe.Node(
-        interface=fsl.ApplyWarp(),
-        name=f"resample_anat_brain_mask_in_standard_{pipe_num}",
+    anat_brain_mask_to_func_res = anat_brain_mask_to_bold_res(
+        wf_name="anat_brain_mask_to_bold_res", cfg=cfg, pipe_num=pipe_num
     )
 
-    anat_brain_mask_to_func_res.inputs.interp = "nn"
-    anat_brain_mask_to_func_res.inputs.premat = cfg.registration_workflows[
-        "anatomical_registration"
-    ]["registration"]["FSL-FNIRT"]["identity_matrix"]
-
     node, out = strat_pool.get_data("space-template_desc-brain_mask")
-    wf.connect(node, out, anat_brain_mask_to_func_res, "in_file")
+    wf.connect(
+        node,
+        out,
+        anat_brain_mask_to_func_res,
+        "inputspec.pace-template_desc-brain_mask",
+    )
 
     wf.connect(
-        anat_brain_to_func_res, "out_file", anat_brain_mask_to_func_res, "ref_file"
+        anat_brain_to_func_res,
+        "outputspec.space-template_res-bold_desc-brain_T1w",
+        anat_brain_mask_to_func_res,
+        "inputspec.space-template_desc-preproc_T1w",
     )
 
     # Resample func mask in template space back to native space
@@ -1574,15 +1655,24 @@ def bold_mask_anatomical_resampled(wf, cfg, strat_pool, pipe_num, opt=None):
     func_mask_template_to_native.inputs.outputtype = "NIFTI_GZ"
 
     wf.connect(
-        anat_brain_mask_to_func_res, "out_file", func_mask_template_to_native, "in_file"
+        anat_brain_mask_to_func_res,
+        "outputspec.space-template_desc-bold_mask",
+        func_mask_template_to_native,
+        "in_file",
     )
 
     node, out = strat_pool.get_data("desc-preproc_bold")
     wf.connect(node, out, func_mask_template_to_native, "master")
 
     outputs = {
-        "space-template_res-bold_desc-brain_T1w": (anat_brain_to_func_res, "out_file"),
-        "space-template_desc-bold_mask": (anat_brain_mask_to_func_res, "out_file"),
+        "space-template_res-bold_desc-brain_T1w": (
+            anat_brain_to_func_res,
+            "outputspec.space-template_res-bold_desc-brain_T1w",
+        ),
+        "space-template_desc-bold_mask": (
+            anat_brain_mask_to_func_res,
+            "outputspec.space-template_desc-bold_mask",
+        ),
         "space-bold_desc-brain_mask": (func_mask_template_to_native, "out_file"),
     }
 

CPAC/nuisance/nuisance.py

@@ -75,8 +75,8 @@ def choose_nuisance_blocks(cfg, rpool, generate_only=False):
     ]
     apply_transform_using = to_template_cfg["apply_transform"]["using"]
     input_interface = {
-        "default": ("desc-preproc_bold", ["desc-preproc_bold", "bold"]),
-        "abcd": ("desc-preproc_bold", "bold"),
+        "default": ("desc-preproc_bold", ["desc-preproc_bold", "desc-reorient_bold"]),
+        "abcd": ("desc-preproc_bold", "desc-reorient_bold"),
         "single_step_resampling_from_stc": ("desc-preproc_bold", "desc-stc_bold"),
     }.get(apply_transform_using)
     if input_interface is not None: