Merge pull request #21 from BIDS-Apps/fs

run recon_all if freesurfer directory not found and others

Author: fliem

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 .idea
 # Byte-compiled / optimized / DLL files
 __pycache__/

Dockerfile

@@ -1,6 +1,5 @@
 FROM bids/base_fsl
 
-
 #### FreeSurfer
 RUN apt-get -y update && \
     wget -qO- https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \
@@ -41,17 +40,13 @@ RUN sudo apt-get update && apt-get install -y python3
 RUN sudo apt-get update && apt-get install -y python3-pip
 RUN pip3 install pandas
 RUN pip3 install pybids
+RUN pip3 install nibabel
+RUN pip3 install joblib
 
-RUN apt-get install -y tree htop
-RUN apt-get install -y tcsh
-RUN apt-get install -y bc
-RUN apt-get install -y tar libgomp1 perl-modules
-
-RUN apt-get install -y curl
-RUN curl -sL https://deb.nodesource.com/setup_6.x | bash -
-RUN apt-get install -y nodejs
-RUN npm install -g bids-validator@0.19.8
-
+RUN sudo apt-get update && apt-get install -y tree htop
+RUN sudo apt-get update && apt-get install -y tcsh
+RUN sudo apt-get update && apt-get install -y bc
+RUN sudo apt-get update && apt-get install -y tar libgomp1 perl-modules
 
 RUN mkdir /scratch
 RUN mkdir /local-scratch
@@ -61,6 +56,14 @@ COPY run.py /code/run.py
 COPY tracula.py /code/tracula.py
 RUN chmod +x /code/run.py
 
+# freesurfer repo
+RUN wget https://github.com/bids-apps/freesurfer/archive/v6.0.0-5.tar.gz && \
+tar xfz v6.0.0-5.tar.gz && rm -r v6.0.0-5.tar.gz && \
+cd freesurfer-6.0.0-5 && mv run.py /code/run_freesurfer.py
+
+RUN touch /code/version
+ENV PATH=/code:$PATH
+
 COPY version /version
 
 ENTRYPOINT ["/code/run.py"]

README.md

@@ -47,17 +47,22 @@ doi: https://doi.org/10.1101/079145*
 ##
 
 ### Data
-Tracula requires one DWI volume and a Freesurfer reconstruction
+Tracula requires one DWI volume and a one T1w image
 per participant (or session if the data is longitudinal).
-The Freesurfer reconstruction should be performed with the
+
+In a first step, the app will run the FreeSurfer reconstruction
+(`recon-all`). If the Freesurfer reconstruction is already available
+and is provided via the `{freesurfer_dir}` argument, this step is skipped.
+In this case `recon-all` should have been performed with the
 [Freesurfer BIDS App](https://github.com/bids-apps/freesurfer)
 (or at least follow the BIDS naming scheme).
 
 ### Analysis levels
 
 - **participant**: Tract reconstruction
 
-    Performs the three steps (prep, bedp, path) of Tracula's `trac-all`,
+    Runs `recon-all` if not already available.
+    Subsequently, performs the three steps (prep, bedp, path) of Tracula's `trac-all`,
     reconstructing major fiber tracts form Freesurfer outputs and
     DWI raw data.
     All data is written into `{output_dir}`.
@@ -79,21 +84,24 @@ The Freesurfer reconstruction should be performed with the
     calculation can be found in the *TMI_info* column of the
     output file.
 
-- **group2**: Overall tract statistics
+- **group2**: Tract statistics
 
-    Collects characteristics of a tract (average FA...)
-    for multiple subjects.
-    Output is written to
-    `{output_dir}/00_group2_tract_stats/{tract_name}_stats.tsv`.
+    Collects tract stats for multiple subjects.
+    Mean stats of a tract (average FA...) are written to
+    `{output_dir}/00_group2_tract_stats/overall_stats/`.
+    Along-tract stats are written to
+    `{output_dir}/00_group2_tract_stats/byvoxel_stats/`.
 
 
 ### Usage
 This App has the following command line arguments:
 
     usage: run.py [-h] --license_key LICENSE_KEY
                   [--participant_label PARTICIPANT_LABEL [PARTICIPANT_LABEL ...]]
+                  [--session_label SESSION_LABEL [SESSION_LABEL ...]]
                   [--freesurfer_dir FREESURFER_DIR]
-                  [--stages {prep,bedp,path,all} [{prep,bedp,path,all} ...]] [-v]
+                  [--stages {prep,bedp,path,all} [{prep,bedp,path,all} ...]]
+                  [--n_cpus N_CPUS] [--run-freesurfer-tests-only] [-v]
                   bids_dir output_dir {participant,group1,group2}
 
     BIDS App for Tracula processing stream.
@@ -108,10 +116,10 @@ This App has the following command line arguments:
                             theparticipant level analysis.
       {participant,group1,group2}
                             Level of the analysis that will be performed.
-                            "participant": reconstructs paths (trac-all -prep,
-                            -bedp and -path), "group1": collects motion stats in
-                            one file, "group2": collects single subject overall
-                            path stats in one file.
+                            "participant": runs FreeSurfer and reconstructs paths
+                            (trac-all -prep, -bedp and -path), "group1": collects
+                            motion stats in one file, "group2": collects tract
+                            stats in one file.
 
     optional arguments:
       -h, --help            show this help message and exit
@@ -128,16 +136,27 @@ This App has the following command line arguments:
                             parameter is not provided all subjects should be
                             analyzed. Multiple participants can be specified with
                             a space separated list. (default: None)
+      --session_label SESSION_LABEL [SESSION_LABEL ...]
+                            The label of the sessions that should be analyzed. The
+                            label corresponds to ses-<session_label> from the BIDS
+                            spec (so it does not include "ses-"). If this
+                            parameter is not provided all sessions should be
+                            analyzed. Multiple sessions can be specified with a
+                            space separated list. (default: None)
       --freesurfer_dir FREESURFER_DIR
-                            The directory with the freesurfer data. If not
-                            specified, output_dir is assumed to be populated with
-                            freesurfer data. (default: None)
+                            The directory with the FreeSurfer data. If not
+                            specified, FreeSurfer data is written into output_dir.
+                            If FreeSurfer data cannot be found for a subject, this
+                            app will run FreeSurfer as well. (default: None)
       --stages {prep,bedp,path,all} [{prep,bedp,path,all} ...]
                             Participant-level trac-all stages to run. Passing"all"
                             will run "prep", "bedp" and "path". (default: ['all'])
+      --n_cpus N_CPUS       Number of CPUs/cores available to use. (default: 1)
+      --run-freesurfer-tests-only
+                            Dev option to enable freesurfer tests on circleci
+                            (default: False)
       -v, --version         show program's version number and exit
 
-
 ##
 ### Examples
 To run it in participant level mode (for one participant):
@@ -175,7 +194,7 @@ To aggregate motion statistics into one file (group1 stage), run:
 
 
 
-To collect single subject overall path stats in one file (group2 stage), run:
+To aggregate tract statistics into one file  (group2 stage), run:
 
         docker run -ti --rm \
          -v /data/ds114/sourcedata:/bids_dataset:ro \

circle.yml

@@ -1,7 +1,9 @@
 general:
   artifacts:
     - "~/outputs1"
+    - "~/outputs1_fs_tests"
     - "~/outputs2"
+    - "~/outputs2_1ses"
     - "~/outputs2_session_label"
 
 machine:
@@ -16,8 +18,9 @@ dependencies:
   override:
     - if [[ ! -d ~/data/ds114_test1 ]]; then wget -c -O ${HOME}/ds114_test1.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/57e54a326c613b01d7d3ed90" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test1.tar -C ${HOME}/data; fi
     - if [[ ! -d ~/data/ds114_test2 ]]; then wget -c -O ${HOME}/ds114_test2.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/57e549f9b83f6901d457d162" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test2.tar -C ${HOME}/data; fi
-    - if [[ ! -d ~/data/ds114_test1_freesurfer ]]; then wget -c -O ${HOME}/ds114_test1_freesurfer.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/5882adf3b83f6901f564da49" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test1_freesurfer.tar -C ${HOME}/data; fi
-    - if [[ ! -d ~/data/ds114_test2_freesurfer ]]; then wget -c -O ${HOME}/ds114_test2_freesurfer.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/5882b0e3b83f6901fb64da18" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test2_freesurfer.tar -C ${HOME}/data; fi
+    - if [[ ! -d ~/data/ds114_test2_1ses ]]; then cp -r ~/data/ds114_test2 ~/data/ds114_test2_1ses && rm -r ~/data/ds114_test2_1ses/sub-*/ses-retest; fi
+    - if [[ ! -d ~/data/ds114_test1_freesurfer ]]; then wget -c -O ${HOME}/ds114_test1_freesurfer.tar "https://osf.io/j6zk2/?action=download&version=1" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test1_freesurfer.tar -C ${HOME}/data && cd ${HOME}/data && mv ds114_test1_freesurfer_precomp_v6.0.0 ds114_test1_freesurfer; fi
+    - if [[ ! -d ~/data/ds114_test2_freesurfer ]]; then wget -c -O ${HOME}/ds114_test2_freesurfer.tar "https://osf.io/yhzzj/?action=download&version=1" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test2_freesurfer.tar -C ${HOME}/data&& cd ${HOME}/data && mv ds114_test2_freesurfer_precomp_v6.0.0 ds114_test2_freesurfer; fi
     - if [[ ! -d ~/data/ds114_test1_tracula_precomp ]]; then wget -c -O ${HOME}/ds114_test1_tracula_precomp.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/5882af28594d9001f447f03a" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test1_tracula_precomp.tar -C ${HOME}/data; fi
     - if [[ ! -d ~/data/ds114_test2_tracula_precomp ]]; then wget -c -O ${HOME}/ds114_test2_tracula_precomp.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/5882b3c3594d9001f047f031" && mkdir -p ${HOME}/data && tar xf ${HOME}/ds114_test2_tracula_precomp.tar -C ${HOME}/data; fi
     - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi
@@ -30,14 +33,17 @@ test:
   override:
     # print version
     - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset -v ${HOME}/data/ds114_test1_freesurfer:/freesurfer bids/${CIRCLE_PROJECT_REPONAME,,} --version
+    # participant level tests for 1ses dataset: cross dataset that has session level with one session
+    - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test2_1ses:/bids_dataset -v ${HOME}/outputs2_1ses:/outputs -v ${HOME}/data/ds114_test1_freesurfer:/freesurfer bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 01 --license_key="~/test.key" --freesurfer_dir /freesurfer --stages prep && cat ${HOME}/outputs2_1ses/sub-01/scripts/trac-preproc.done :
+        timeout: 21600
     # participant level tests for single session dataset
     - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset -v ${HOME}/outputs1:/outputs -v ${HOME}/data/ds114_test1_freesurfer:/freesurfer bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 01 --license_key="~/test.key" --freesurfer_dir /freesurfer --stages prep && cat ${HOME}/outputs1/sub-01/scripts/trac-preproc.done :
         timeout: 21600
     # group level test for single session dataset: group1
     - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset -v ${HOME}/data/ds114_test1_tracula_precomp:/outputs -v ${HOME}/data/ds114_test1_freesurfer:/freesurfer bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs group1 --license_key="~/test.key" --freesurfer_dir /freesurfer && mkdir -p ${HOME}/outputs1/ && sudo mv ${HOME}/data/ds114_test1_tracula_precomp/00_group* ${HOME}/outputs1/ && cat ${HOME}/outputs1/00_group1_motion_stats/group_motion.tsv :
         timeout: 21600
     # group level test for single session dataset: group2
-    - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset -v ${HOME}/data/ds114_test1_tracula_precomp:/outputs -v ${HOME}/data/ds114_test1_freesurfer:/freesurfer bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs group2 --license_key="~/test.key" --freesurfer_dir /freesurfer && mkdir -p ${HOME}/outputs1/ && sudo mv ${HOME}/data/ds114_test1_tracula_precomp/00_group* ${HOME}/outputs1/ && cat ${HOME}/outputs1/00_group2_tract_stats/fminor_stats.tsv :
+    - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset -v ${HOME}/data/ds114_test1_tracula_precomp:/outputs -v ${HOME}/data/ds114_test1_freesurfer:/freesurfer bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs group2 --license_key="~/test.key" --freesurfer_dir /freesurfer && mkdir -p ${HOME}/outputs1/ && sudo mv ${HOME}/data/ds114_test1_tracula_precomp/00_group* ${HOME}/outputs1/ && cat ${HOME}/outputs1/00_group2_tract_stats/overall_stats/fminor_stats.tsv && cat ${HOME}/outputs1/00_group2_tract_stats/byvoxel_stats/lh.cab_PP.avg33_mni_bbr.FA.txt:
         timeout: 21600
     # participant level tests for a longitudinal dataset
     - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test2:/bids_dataset -v ${HOME}/outputs2:/outputs -v ${HOME}/data/ds114_test2_freesurfer:/freesurfer bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 01 --license_key="~/test.key" --freesurfer_dir /freesurfer --stages prep && cat ${HOME}/outputs2/sub-01_ses-test.long.sub-01/scripts/trac-preproc.done :
@@ -49,9 +55,11 @@ test:
     - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test2:/bids_dataset -v ${HOME}/data/ds114_test2_tracula_precomp:/outputs -v ${HOME}/data/ds114_test2_freesurfer:/freesurfer bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs group1 --license_key="~/test.key" --freesurfer_dir /freesurfer && mkdir -p ${HOME}/outputs2/ && sudo mv ${HOME}/data/ds114_test2_tracula_precomp/00_group* ${HOME}/outputs2/ && cat ${HOME}/outputs2/00_group1_motion_stats/group_motion.tsv :
         timeout: 21600
     # group level test for a longitudinal dataset: group2
-    - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test2:/bids_dataset -v ${HOME}/data/ds114_test2_tracula_precomp:/outputs -v ${HOME}/data/ds114_test2_freesurfer:/freesurfer bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs group2 --license_key="~/test.key" --freesurfer_dir /freesurfer && mkdir -p ${HOME}/outputs2/ && sudo mv ${HOME}/data/ds114_test2_tracula_precomp/00_group* ${HOME}/outputs2/ && cat ${HOME}/outputs2/00_group2_tract_stats/fminor_stats.tsv:
+    - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test2:/bids_dataset -v ${HOME}/data/ds114_test2_tracula_precomp:/outputs -v ${HOME}/data/ds114_test2_freesurfer:/freesurfer bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs group2 --license_key="~/test.key" --freesurfer_dir /freesurfer && mkdir -p ${HOME}/outputs2/ && sudo mv ${HOME}/data/ds114_test2_tracula_precomp/00_group* ${HOME}/outputs2/ && cat ${HOME}/outputs2/00_group2_tract_stats/overall_stats/fminor_stats.tsv && cat ${HOME}/outputs2/00_group2_tract_stats/byvoxel_stats/lh.cab_PP.avg33_mni_bbr.FA.txt :
+        timeout: 21600
+    # starts freesurfer processing only
+    - mkdir -p ${HOME}/outputs1_fs_tests/tracula && mkdir -p ${HOME}/outputs1_fs_tests/freesurfer && docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset -v ${HOME}/outputs1_fs_tests:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs/tracula participant --participant_label 01 --license_key="~/test.key" --freesurfer_dir /outputs/freesurfer --run-freesurfer-tests-only && cat  ${HOME}/outputs1_fs_tests/freesurfer/sub-01/scripts/recon-all.done :
         timeout: 21600
-
 deployment:
   hub:
     owner: BIDS-Apps

full_tests/run_full_tests_v6.0.0-3.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+# cd /data.nfs/ds114/tracula_full_tests
+# screen -L bash run_full_tests_....sh ds114_test1 and
+# screen -L bash run_full_tests_....sh ds114_test2
+
+ds_name=$1
+echo Running ${ds_name}
+
+tracula_version=v6.0.0-3
+wd=/data.nfs/ds114/tracula_full_tests/${tracula_version}
+
+data_dir=${wd}/data
+out_root_dir=${wd}/out
+
+mkdir -p $data_dir
+mkdir -p $out_root_dir
+
+if [[ ! -d ${data_dir}/ds114_test1 ]]; then wget -c -O ${data_dir}/ds114_test1.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/57e54a326c613b01d7d3ed90" && tar xf ${data_dir}/ds114_test1.tar -C ${data_dir}; fi
+if [[ ! -d ${data_dir}/ds114_test2 ]]; then wget -c -O ${data_dir}/ds114_test2.tar "https://files.osf.io/v1/resources/9q7dv/providers/osfstorage/57e549f9b83f6901d457d162" && tar xf ${data_dir}/ds114_test2.tar -C ${data_dir}; fi
+if [[ ! -d ${data_dir}/ds114_test1_freesurfer_precomp_v6.0.0 ]]; then wget -c -O ${data_dir}/ds114_test1_freesurfer.tar "https://osf.io/j6zk2/?action=download&version=1" && tar xf ${data_dir}/ds114_test1_freesurfer.tar -C ${data_dir}; fi
+if [[ ! -d ${data_dir}/ds114_test2_freesurfer_precomp_v6.0.0 ]]; then wget -c -O ${data_dir}/ds114_test2_freesurfer.tar "https://osf.io/yhzzj/?action=download&version=1" && tar xf ${data_dir}/ds114_test2_freesurfer.tar -C ${data_dir}; fi
+if [[ -e ${data_dir}/*.tar ]]; then rm -r ${data_dir}/*.tar; fi
+
+
+
+
+in_dir=${data_dir}/${ds_name}
+fs_dir=${data_dir}/${ds_name}_freesurfer_precomp_v6.0.0
+out_dir=${out_root_dir}/${ds_name}
+mkdir -p $out_dir
+chmod -R 777 $out_dir
+cd $out_dir
+
+docker run --rm -ti \
+-v ${in_dir}:/data/in \
+-v ${fs_dir}:/data/fs \
+-v ${out_dir}:/data/out \
+bids/tracula:${tracula_version} \
+/data/in /data/out participant \
+--license_key xxx \
+--freesurfer_dir /data/fs
+
+
+docker run --rm -ti \
+-v ${in_dir}:/data/in \
+-v ${fs_dir}:/data/fs \
+-v ${out_dir}:/data/out \
+bids/tracula:${tracula_version} \
+/data/in /data/out group1 \
+--license_key xxx \
+--freesurfer_dir /data/fs
+
+
+
+docker run --rm -ti \
+-v ${in_dir}:/data/in \
+-v ${fs_dir}:/data/fs \
+-v ${out_dir}:/data/out \
+bids/tracula:${tracula_version} \
+/data/in /data/out group2 \
+--license_key xxx \
+--freesurfer_dir /data/fs
+
+tar -zcvf ${wd}/results_tracula_${tracula_version}_${ds_name}.tar.gz ${out_dir}
+