🚚 Move variant-lite.Singularity → Singularity.latest-lite (#1464)

Author: shnizzedy

.circleci/config.yml

@@ -1,6 +1,42 @@
 version: 2.1
 
 commands:
+  build-images:
+    parameters:
+      variant:
+        type: string
+        default: ""
+    steps:
+      - set-up-variant:
+          variant: "<< parameters.variant >>"
+      - run:
+          name: "Building Docker image"
+          command: |
+            docker build -t fcpindi/c-pac:${DOCKER_TAG} . ${DOCKERFILE}
+            docker save fcpindi/c-pac:${DOCKER_TAG} | gzip > ${DOCKER_TARBALL}.tar.gz
+          no_output_timeout: 5h
+      - run:
+          name: "Starting local registry"
+          command: docker run -d -p 5000:5000 --restart=always --name registry registry:2
+      - install-singularity-requirements
+      - when:
+          condition:
+            or:
+            - equal: [ "", "<< parameters.variant >>" ]
+            - equal: [ "lite", "<< parameters.variant >>" ]
+          steps:
+            - run: 
+                name: "Cloning Singularity 2.5.2"
+                command: git clone -b 2.5.2 https://github.com/sylabs/singularity
+      - set-up-singularity
+      - run:
+          name: "Building Singularity image from Docker image"
+          command: |
+            docker load < ${DOCKER_TARBALL}.tar.gz
+            docker tag fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT} localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
+            docker push localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
+            SINGULARITY_NOHTTPS=1 singularity build cpac-singularity-image${VARIANT}.simg docker://localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
+          no_output_timeout: 5h
   check-for-built-images:
     steps:
       - run:
@@ -10,6 +46,25 @@ commands:
             then
               circleci step halt
             fi
+  configure-git-user:
+    steps:
+      - add_ssh_keys:
+          fingerprints:
+            - "12:bc:f2:e4:31:cc:72:54:54:bc:f5:5b:89:e6:d8:ee"
+      - run:
+          name: "Configuring git user"
+          command: |
+            sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 78BD65473CB3BD13
+            curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
+            sudo apt-get update
+            sudo apt-get install git -y
+            git config --global user.email "${CIRCLE_USERNAME}@users.noreply.github.com"
+            git config --global user.name "${CIRCLE_USERNAME} @ CircleCI"
+  get-sample-bids-data:
+    steps:
+      - run:
+          name: Getting Sample BIDS Data
+          command: git clone https://github.com/bids-standard/bids-examples.git
   set-python-version:
     steps:
       - run:
@@ -24,6 +79,14 @@ commands:
           name: "Installing Singularity requirements"
           command: |
            sudo apt-get update && sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.1 libarchive-dev libgpgme11-dev libseccomp-dev -y
+  run-pytest-docker:
+    steps:
+      - run:
+          name: Running pytest on Docker image
+          command: |
+            docker load < cpac-docker-image.tar.gz
+            docker run -dit -P -v /home/circleci/project/test-results:/code/test-results -v /home/circleci/project/htmlcov:/code/htmlcov --entrypoint=/bin/bash --name docker_test fcpindi/c-pac:${DOCKER_TAG}
+            docker exec docker_test /bin/bash ./code/dev/circleci_data/test_in_image.sh
   set-up-singularity:
     steps:
       - run:
@@ -35,11 +98,10 @@ commands:
            make
            sudo make install
            cd ..
-  build-images:
+  set-up-variant:
     parameters:
       variant:
         type: string
-        default: ""
     steps:
       - run:
           name: "Setting up variant"
@@ -48,81 +110,50 @@ commands:
             if [[ -n "<< parameters.variant >>" ]]
             then
               DOCKERFILE="variant-<< parameters.variant >>.Dockerfile"
-              sed -i "s|:latest|:${CIRCLE_BRANCH//\//_}|g" ${DOCKERFILE}
+              if [[ -f ${DOCKERFILE} ]]
+              then
+                sed -i "s|:latest|:${CIRCLE_BRANCH//\//_}|g" ${DOCKERFILE}
+              fi
               DOCKER_TARBALL="${DOCKER_TARBALL}-<< parameters.variant >>"
               VARIANT="-<< parameters.variant >>"
               echo "export DOCKERFILE=\"-f ${DOCKERFILE}\"" >> $BASH_ENV
             fi
             echo "export DOCKER_TARBALL=${DOCKER_TARBALL}" >> $BASH_ENV
             echo "export VARIANT=${VARIANT}" >> $BASH_ENV
-      - run:
-          name: "Building Docker image"
-          command: |
-            docker build -t fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT} . ${DOCKERFILE}
-            docker save fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT} | gzip > ${DOCKER_TARBALL}.tar.gz
-          no_output_timeout: 5h
-          # Persist the specified paths (workspace/echo-output) into the workspace for use in downstream job.
-      - run:
-          name: "Starting local registry"
-          command: docker run -d -p 5000:5000 --restart=always --name registry registry:2
-      - install-singularity-requirements
-      - when:
-          condition:
-            equal: [ "", "<< parameters.variant >>" ]
-          steps:
-            - run: 
-                name: "Cloning Singularity 2.5.2"
-                command: git clone -b 2.5.2 https://github.com/sylabs/singularity
-      - set-up-singularity
-      - run:
-          name: "Building Singularity image from Docker image"
-          command: |
-            docker load < ${DOCKER_TARBALL}.tar.gz
-            docker tag fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT} localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
-            docker push localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
-            SINGULARITY_NOHTTPS=1 singularity build cpac-singularity-image${VARIANT}.simg docker://localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}${VARIANT}
-          no_output_timeout: 5h
-  configure-git-user:
+            DOCKER_TAG=${CIRCLE_BRANCH//\//_}${VARIANT}
+            echo "export DOCKER_TAG=${DOCKER_TAG}" >> $BASH_ENV
+  test-singularity-installation:
     steps:
-      - add_ssh_keys:
-          fingerprints:
-            - "12:bc:f2:e4:31:cc:72:54:54:bc:f5:5b:89:e6:d8:ee"
       - run:
-          name: "Configuring git user"
+          name: Testing Singularity installation
           command: |
-            sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 78BD65473CB3BD13
-            curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
-            sudo apt-get update
-            sudo apt-get install git -y
-            git config --global user.email "${CIRCLE_USERNAME}@users.noreply.github.com"
-            git config --global user.name "${CIRCLE_USERNAME} @ CircleCI"
+            pip install -r dev/circleci_data/requirements.txt
+            coverage run -m pytest --junitxml=test-results/junit.xml --continue-on-collection-errors  dev/circleci_data/test_install.py
 
 jobs:
   pytest-docker:
+    parameters:
+      variant:
+        type: string
+        default: ""
     machine:
       image: ubuntu-2004:202010-01
     steps:
       - attach_workspace:
-          # Must be absolute path or relative path from working_directory
           at: /home/circleci/
+      - set-up-variant:
+          variant: "<< parameters.variant >>"
       - check-for-built-images
       - set-python-version
-      - run:
-          name: Getting Sample BIDS Data
-          command: git clone https://github.com/bids-standard/bids-examples.git
-      - run:
-          name: Running pytest on Docker image
-          command: |
-            docker load < cpac-docker-image.tar.gz
-            docker run -dit -P -v /home/circleci/project/test-results:/code/test-results -v /home/circleci/project/htmlcov:/code/htmlcov --entrypoint=/bin/bash --name docker_test fcpindi/c-pac:${CIRCLE_BRANCH//\//_}
-            docker exec docker_test /bin/bash ./code/dev/circleci_data/test_in_image.sh
+      - get-sample-bids-data
+      - run-pytest-docker
       - store_test_results:
           path: test-results
       - store_artifacts:
           path: test-results
       - store_artifacts:
           path: htmlcov
-          no_output_timeout: 5h
+
   pytest-singularity:
     machine:
       image: ubuntu-2004:202010-01
@@ -134,11 +165,8 @@ jobs:
       - set-python-version
       - install-singularity-requirements
       - set-up-singularity
-      - run:
-          name: Testing Singularity installation
-          command: |
-            pip install -r dev/circleci_data/requirements.txt
-            coverage run -m pytest --junitxml=test-results/junit.xml --continue-on-collection-errors  dev/circleci_data/test_install.py
+      - test-singularity-installation
+
   build:
     machine:
       image: ubuntu-2004:202010-01
@@ -196,11 +224,24 @@ workflows:
   version: 2
   build-and-test:
     jobs:
-      - build
-      - build-lite
+      - build:
+          name: "Build images"
+      - build-lite:
+          name: "Build lite variant images"
+      - pytest-docker:
+          name: "Test in Docker"
+          requires:
+            - "Build images"
       - pytest-docker:
+          name: "Test lite variant in Docker"
+          requires:
+            - "Build lite variant images"
+          variant: lite
+      - pytest-singularity:
+          name: "Test in Singularity"
           requires:
-            - build
+            - "Build images"
       - pytest-singularity:
+          name: "Test lite variant in Singularity"
           requires:
-            - build
+            - "Build lite variant images"