Jenkinsfile.unittests

pipeline {
  environment {
    DOCKER_IMAGE = "ci-devel:${BRANCH_NAME}.${BUILD_ID}"
    DOCKER_PARAMS = "-v ${WORKSPACE}:/app -v ${WORKSPACE}/build:/app/build -v /var/lib/jenkins/.ccache:/ccache --runtime nvidia"
  }
  agent { node { label 'gpu' } }
  stages {
    stage('Init') {
      steps {
        script {
            def common = load("ci/Jenkinsfile.common")
            common.cancelPreviousBuilds()
        }
        sh 'printenv | sort'
      }
    }
    stage('Prepare docker image') {
      steps {
        script {
          docker.build(env.DOCKER_IMAGE, "-f ci/devel.Dockerfile --no-cache .")
        }
        // post can't access to ${env.XX} so we have to use stupid hack, thx Jenkins...
        sh '''echo ${DOCKER_IMAGE} > docker-image-name'''
      }
    }
    stage('Installing prebuilt cache') {
      when {
       expression { !env.CHANGE_ID || pullRequest.labels.findAll { it == "ci:full-build" }.size() == 0 }
      }
      steps {
        script {
          sh '''
          prebuilt_version=$(awk '/^lastSuccessfulBuild/{print $2}' /var/lib/jenkins/jobs/deepdetect-prebuilt-cache/branches/master/builds/permalinks)
          # Create a copy on write filesystem instead of copying data
          mkdir -p build-upper build-work build
          sudo mount -t overlay overlay -o lowerdir=/var/lib/jenkins/jobs/deepdetect-prebuilt-cache/branches/master/builds/${prebuilt_version}/archive/build,upperdir=$(pwd)/build-upper,workdir=$(pwd)/build-work $(pwd)/build
          '''
        }
      }
    }
    stage('Configure && Build') {
      steps {
        sh '''mkdir -p build/tmp'''
        script {
          docker.image(env.DOCKER_IMAGE).inside(env.DOCKER_PARAMS) {
            sh '''

cd clients/python/
tox -e pep8,py310
cd ../..

export CCACHE_DIR=/ccache
export PATH="/usr/lib/ccache/:$PATH"
export TMPDIR=$(pwd)/build/tmp
cd build
cmake .. \
    -DBUILD_TESTS=ON  \
    -DBUILD_SPDLOG=ON \
    -DUSE_HTTP_SERVER_OATPP=ON \
    -DUSE_CUDNN=ON  \
    -DUSE_FAISS=ON  \
    -DUSE_SIMSEARCH=ON  \
    -DUSE_CAFFE=ON \
    -DUSE_TSNE=ON  \
    -DUSE_XGBOOST=ON  \
    -DUSE_TORCH=ON  \
    -DUSE_NCNN=ON  \
    -DUSE_TENSORRT=OFF  \
    -DUSE_TENSORRT_OSS=OFF  \
    -DCUDA_ARCH_FLAGS="-gencode arch=compute_61,code=sm_61 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_86,code=sm_86" \
    -DCUDA_ARCH="6.1;7.5;8.6"

make clang-format-check

if [ -f spdlog/src/spdlog-stamp/spdlog-done ]
then
    touch CMakeFiles/cache_present
fi

if [ -f CMakeFiles/cache_present ]
then
    for lib in protobuf spdlog caffe_dd pytorch pytorch_vision Multicore-TSNE ncnn xgboost oatpp oatpp-swagger oatpp-zlib 
    do
        touch "${lib}/src/${lib}-stamp/${lib}-mkdir"*
        touch "${lib}/src/${lib}-stamp/${lib}-download"*
        touch "${lib}/src/${lib}-stamp/${lib}-patch"*
        touch "${lib}/src/${lib}-stamp/${lib}-update"*
        touch "${lib}/src/${lib}-stamp/${lib}-urlinfo.txt"*
        touch "${lib}/src/${lib}-stamp/download-${lib}.cmake"
        touch "${lib}/src/${lib}-stamp/extract-${lib}.cmake"
        touch "${lib}/src/${lib}-stamp/verify-${lib}.cmake"
    done
fi

if [ -f CMakeFiles/cache_present ]
then
    for lib in spdlog protobuf oatpp caffe_dd Multicore-TSNE ncnn xgboost oatpp-swagger oatpp-zlib pytorch pytorch_vision
    do
        touch "${lib}/src/${lib}-stamp/"*
        touch "CMakeFiles/${lib}-complete"
    done
fi

schedtool -B -n 1 -e ionice -n 1 make -j 6
ccache -s
'''
          }
        }
      }
    }
    stage('Tests GPU') {
      when {
       expression { !env.CHANGE_ID || pullRequest.labels.findAll { it == "ci:skip-tests" }.size() == 0 }
      }
      steps {
        lock(resource: null, label: "${NODE_NAME}-gpu", variable: 'LOCKED_GPU', quantity: 1) {
          script {
            docker.image(env.DOCKER_IMAGE).inside(env.DOCKER_PARAMS) {
              sh '''
              export CUDA_VISIBLE_DEVICES=$(echo ${LOCKED_GPU} | sed -n -e "s/[^,]* GPU \\([^[0-9,]]\\)*/\\1/gp")
              echo "****************************"
              echo
              python3 -c 'import torch, sys; c=torch.cuda.device_count() ; print(f"CUDA VISIBLE GPU: {c}"); sys.exit(bool(c == 0 ))'
              echo
              echo "****************************"
              cd build && ctest -V -E "multigpu"
              '''
            }
          }
        }
      }
    }
    stage('Tests multi-GPU') {
      when {
       expression { !env.CHANGE_ID || pullRequest.labels.findAll { it == "ci:skip-tests" }.size() == 0 }
      }
      steps {
        lock(resource: null, label: "${NODE_NAME}-gpu", variable: 'LOCKED_GPU', quantity: 2) {
          script {
            docker.image(env.DOCKER_IMAGE).inside(env.DOCKER_PARAMS) {
              sh '''
              export CUDA_VISIBLE_DEVICES=$(echo ${LOCKED_GPU} | sed -n -e "s/[^,]* GPU \\([^[0-9,]]\\)*/\\1/gp")
              echo "****************************"
              echo
              python3 -c 'import torch, sys; c=torch.cuda.device_count() ; print(f"CUDA VISIBLE GPU: {c}"); sys.exit(bool(c < 2))'
              echo
              echo "****************************"
              cd build && ctest -V -R "multigpu"
              '''
            }
          }
        }
      }
    }
  }
  post {
    always {
      catchError {
        sh '''[ -f docker-image-name ] && docker image rm $(cat docker-image-name)'''
      }
      catchError {
        sh '''sudo umount $(pwd)/build || true'''
      }
      cleanWs(cleanWhenAborted: true, cleanWhenFailure: true, cleanWhenNotBuilt: true, cleanWhenSuccess: true, cleanWhenUnstable: true, cleanupMatrixParent: true, deleteDirs: true)
    }
    success {
      catchError {
        rocketSend(channel: 'build', message: 'Build succeed' ,color: 'green' )
      }
    }
    aborted {
      catchError {
        rocketSend(channel: 'build', message: 'Build superseded or aborted')
      }
    }
    unstable {
      catchError {
        rocketSend(channel: 'build', message: 'Build failed', color: 'red')
      }
    }
    failure {
      catchError {
        rocketSend(channel: 'build', message: 'Build failed', color: 'red')
      }
    }
  }
}