Merge pull request #16 from ntraut/skip

allow to skip not segmented subjects

Author: fliem

.circleci/config.yml

@@ -57,6 +57,15 @@ jobs:
             else
               echo "ds114_test2_freesurfer was cached"
             fi
+      - run:
+          name: Create a FreeSurfer folder with subject 02 missing
+          command: |
+            if [[ ! -d /tmp/data/ds114_test1_freesurfer_missing ]]; then
+              mkdir /tmp/data/ds114_test1_freesurfer_missing
+              cp -R /tmp/data/ds114_test1_freesurfer/sub-01 /tmp/data/ds114_test1_freesurfer_missing
+            else
+              echo "ds114_test1_freesurfer_missing was cached"
+            fi
       - save_cache:
          key: data-v3-{{ .Revision }}-{{ epoch }}
          paths:
@@ -269,6 +278,31 @@ jobs:
           command: |
             cat ${OUTDIR}/ds114_test2/baracus/00_group/group_predicted_age.tsv
 
+      # Test 5: skip_missing
+      - run:
+          name: Test 5 skip_missing - 1
+          command: |
+            docker run --rm -ti -v ${DATADIR}/ds114_test1:/data/in \
+            -v ${DATADIR}/ds114_test1_freesurfer_missing:/data/fs \
+            -v ${OUTDIR}/ds114_test1_missing:/data/out \
+            bids/${CIRCLE_PROJECT_REPONAME}:latest \
+            /data/in /data/out participant --freesurfer_dir /data/fs \
+            --license_key="~/test.key" --skip_missing
+      - run:
+          name: Test 5 skip_missing - 2
+          command: |
+            docker run --rm -ti -v ${DATADIR}/ds114_test1:/data/in \
+            -v ${DATADIR}/ds114_test1_freesurfer_missing:/data/fs \
+            -v ${OUTDIR}/ds114_test1_missing:/data/out \
+            bids/${CIRCLE_PROJECT_REPONAME}:latest \
+            /data/in /data/out group --freesurfer_dir /data/fs \
+            --license_key="~/test.key" --skip_missing
+      - run:
+          name: Test 5 skip_missing - 3
+          command: |
+            cat ${OUTDIR}/ds114_test1_missing/baracus/00_group/group_predicted_age.tsv
+
+
       - store_artifacts:
           path: /tmp/outputs/
 

README.md

@@ -139,6 +139,8 @@ previously processed data:
                             found, recon-all will be run. If not specified
                             freesurfer data will be saved to {out_dir}/freesurfer
       --models {Liem2016__OCI_norm,Liem2016__full_2samp_training} [{Liem2016__OCI_norm,Liem2016__full_2samp_training} ...]
+      --skip_missing
+                            Flag to skip not segmented subjects
       --license_key LICENSE_KEY
                             FreeSurfer license key - letters and numbers after "*"
                             in the email you received after registration. To

baracus/prepare.py

@@ -18,10 +18,11 @@ def downsample_surfs(fs_dir, out_dir, subject, hemis=["lh", "rh"], meas=["thickn
                 os.makedirs(subject_dir)
             out_file = os.path.join(subject_dir, "{h}.{m}.mgh".format(h=h, m=m))
             out_files[h + "_" + m + "_file"] = out_file
-
-            cmd = "mris_preproc --s {subject} --target fsaverage4 --hemi {h} " \
-                  "--meas {m} --out {out_file}".format(subject=subject, h=h, m=m, out_file=out_file)
-            run(cmd, env={"SUBJECTS_DIR": fs_dir})
+            if not os.path.isfile(out_file):
+                cmd = "mris_preproc --s {subject} --target fsaverage4 --hemi {h} " \
+                    "--meas {m} --out {out_file}".format(
+                        subject=subject, h=h, m=m, out_file=out_file)
+                run(cmd, env={"SUBJECTS_DIR": fs_dir})
     return out_files
 
 
@@ -30,14 +31,16 @@ def prepare_aseg(fs_dir, out_dir, subject):
     if not os.path.isdir(subject_dir):
         os.makedirs(subject_dir)
     out_file = os.path.join(subject_dir, "aseg")
-    cmd = "asegstats2table --subjects {subject} --meas volume --tablefile {out_file}".format(subject=subject,
-                                                                                             out_file=out_file)
-    print(cmd)
-    run(cmd, env={"SUBJECTS_DIR": fs_dir})
+    if not os.path.isfile(out_file):
+        cmd = "asegstats2table --subjects {subject} --meas volume --tablefile {out_file}".format(subject=subject,
+                                                                                                 out_file=out_file)
+        print(cmd)
+        run(cmd, env={"SUBJECTS_DIR": fs_dir})
     return out_file
 
 
-def run_prepare_all(bids_dir, freesurfer_dir, out_dir, subjects_to_analyze, sessions_to_analyze, n_cpus, license_key):
+def run_prepare_all(bids_dir, freesurfer_dir, out_dir, subjects_to_analyze, sessions_to_analyze, n_cpus, license_key,
+                    skip_missing=False):
     """
 
     :param bids_dir:
@@ -58,7 +61,7 @@ def run_prepare_all(bids_dir, freesurfer_dir, out_dir, subjects_to_analyze, sess
     for subject in subjects_to_analyze:
         sessions = sessions_to_analyze.get(subject)
         freesurfer_subjects.extend(run_fs_if_not_available(bids_dir, freesurfer_dir, subject, license_key, n_cpus,
-                                                       sessions))
+                                                       sessions, skip_missing))
 
     # downsample surfaces to fsaverage4 and extract subcortical data from aseg
     out_files = {}

baracus/utils.py

@@ -19,7 +19,7 @@ def run(command, env={}, ignore_errors=False):
         raise Exception("Non zero return code: %d" % process.returncode)
 
 
-def run_fs_if_not_available(bids_dir, freesurfer_dir, subject_label, license_key, n_cpus, sessions=[]):
+def run_fs_if_not_available(bids_dir, freesurfer_dir, subject_label, license_key, n_cpus, sessions=[], skip_missing=False):
     freesurfer_subjects = []
     if sessions:
         # long
@@ -32,21 +32,32 @@ def run_fs_if_not_available(bids_dir, freesurfer_dir, subject_label, license_key
     fs_missing = False
     for fss in freesurfer_subjects:
         if not os.path.exists(os.path.join(freesurfer_dir, fss, "scripts/recon-all.done")):
-            fs_missing = True
+            if skip_missing:
+                freesurfer_subjects.remove(fss)
+            else:
+                fs_missing = True
 
-        if fs_missing:
-            cmd = "run_freesurfer.py {in_dir} {out_dir} participant " \
-                  "--hires_mode disable " \
-                  "--participant_label {subject_label} " \
-                  "--license_key {license_key} " \
-                  "--n_cpus {n_cpus} --steps cross-sectional".format(in_dir=bids_dir,
-                                                                     out_dir=freesurfer_dir,
-                                                                     subject_label=subject_label,
-                                                                     license_key=license_key,
-                                                                     n_cpus=n_cpus)
+    if fs_missing:
+        cmd = "run_freesurfer.py {in_dir} {out_dir} participant " \
+              "--hires_mode disable " \
+              "--participant_label {subject_label} " \
+              "--license_key {license_key} " \
+              "--n_cpus {n_cpus} --steps cross-sectional".format(in_dir=bids_dir,
+                                                                 out_dir=freesurfer_dir,
+                                                                 subject_label=subject_label,
+                                                                 license_key=license_key,
+                                                                 n_cpus=n_cpus)
 
-            print("Freesurfer for {} not found. Running recon-all: {}".format(subject_label, cmd))
-            run(cmd)
+        print("Freesurfer for {} not found. Running recon-all: {}".format(subject_label, cmd))
+        run(cmd)
+
+    for fss in freesurfer_subjects:
+        aseg_file = os.path.join(freesurfer_dir, fss, "stats/aseg.stats")
+        if not os.path.isfile(aseg_file):
+            if skip_missing:
+                freesurfer_subjects.remove(fss)
+            else:
+                raise FileNotFoundError(aseg_file)
     return freesurfer_subjects
 
 

scripts/run_brain_age_bids.py

@@ -38,6 +38,7 @@
                                                  "out_dir}/freesurfer")
     parser.add_argument('--models', choices=models_list, default=["Liem2016__OCI_norm"], help='',
                         nargs="+")
+    parser.add_argument('--skip_missing', help='Flag to skip not segmented subjects', action='store_true')
     parser.add_argument('--license_key',
                         help='FreeSurfer license key - letters and numbers after "*" in the email you '
                              'received after registration. To register (for free) visit '
@@ -72,7 +73,7 @@
     if args.analysis_level == "participant":
 
         data_files = run_prepare_all(args.bids_dir, freesurfer_dir, out_dir, subjects_to_analyze,
-                                     sessions_to_analyze, args.n_cpus, args.license_key)
+                                     sessions_to_analyze, args.n_cpus, args.license_key, args.skip_missing)
 
         for subject, d in data_files.items():
             d["out_dir"] = out_dir
@@ -86,7 +87,10 @@
         df = pd.DataFrame([])
         for subject in freesurfer_subjects_to_analyze:
             in_file = os.path.join(out_dir, subject, subject + "_predicted_age.tsv")
-            df = df.append(pd.read_csv(in_file, sep="\t"))
+            if os.path.isfile(in_file):
+                df = df.append(pd.read_csv(in_file, sep="\t"))
+            elif not args.skip_missing:
+                raise FileNotFoundError(in_file)
 
         group_out_dir = os.path.join(out_dir, "00_group")
         if not os.path.isdir(group_out_dir):