.gitlab-ci.yml

stages:
    - pre_build
    - build 
    - test
    - performance_eval
        
ci_builder_pre:
    stage: pre_build
    tags:
        - ci_testing_cc
    variables:
        GIT_STRATEGY: clone
    before_script:
        - git branch -vva
        - echo $commit_author_name
    script:
        - echo "checking availability of ci_testing_cc"
        - git --version
        - git submodule set-url submodules/xdmfwriter https://github.com/TUM-I5/XdmfWriter.git
        - git submodule set-url submodules/cxxtest https://github.com/CxxTest/cxxtest.git
        - git submodule set-url submodules/utils https://github.com/TUM-I5/utils.git
        - git submodule set-url submodules/async https://github.com/TUM-I5/ASYNC.git
        - git submodule set-url submodules/scons-tools https://github.com/TUM-I5/scons-tools.git
        - git submodule set-url submodules/PUML https://github.com/TUM-I5/PUML2.git
        - git submodule set-url submodules/easi https://github.com/SeisSol/easi.git
        - git submodule set-url submodules/yaml-cpp https://github.com/jbeder/yaml-cpp.git  
        - git submodule set-url submodules/ImpalaJIT https://github.com/uphoffc/ImpalaJIT.git
        - git submodule set-url submodules/yateto https://github.com/SeisSol/yateto.git
        - git submodule set-url submodules/eigen3 https://github.com/eigenteam/eigen-git-mirror
        - git submodule set-url submodules/Device https://github.com/SeisSol/Device.git
        - git submodule --quiet update --init --recursive 
        - wget -q https://syncandshare.lrz.de/dl/fiJNAokgbe2vNU66Ru17DAjT/netcdf-4.6.1.tar.gz
        - wget -q https://github.com/hfp/libxsmm/archive/master.zip
        - sudo apt-get install -qq --yes gcc-8 g++-8 gfortran-8 pkg-config cmake cmake-data 
        - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 100
        - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 100
        - sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-8 100
        - sudo apt-get install -qq --yes unzip openmpi-bin openmpi-common libopenmpi-dev hdf5-tools libhdf5-openmpi-dev python3 python3-pip libmetis-dev libparmetis-dev cxxtest libarmadillo-dev
        - gfortran --version
        - pip3 install --upgrade pip
        - pip3 install 'numpy>=1.12.0'
        - pip3 install lxml
        - pip3 install setuptools
        - pip3 install sphinx
        - pip3 install sphinx_rtd_theme
        - tar -xzf netcdf-4.6.1.tar.gz
        - cd netcdf-4.6.1 && CC=h5pcc ./configure --prefix=/usr --enable-shared=no --disable-dap && make && sudo make install && cd ..
        - unzip master.zip
        - cd libxsmm-master && make generator && sudo cp bin/libxsmm_gemm_generator /usr/bin && cd ..
        - git clone --depth 1 --branch v1.0.0 https://github.com/SeisSol/easi.git && mkdir easi_build && cd easi_build && cmake -DASAGI=OFF -DIMPALAJIT=OFF -DCMAKE_INSTALL_PREFIX=$HOME ../easi && make -j $(nproc) install && cd ..
        
ci_builder_build:
    stage: build
    tags:
        - ci_testing_cc
    needs:
        - job: ci_builder_pre
    before_script:
        - cat .git/config
    parallel:
        matrix:
             - precision: double
             - precision: single
    script:
        - echo "now building"
        - echo $CI_REGISTRY $CI_REGISTRY_IMAGE $CI_REGISTRY_USER
        - git submodule update --init --recursive
        - cmake --version
        - set -euo pipefail
        - export IFS=$'\n\t'
        - export CTEST_OUTPUT_ON_FAILURE=1
        - equation=elastic
        - mechanisms=0
        - for build_type in Release ; 
             do for equation in elastic ;
                 do dirname="build_${equation}_${precision}_${build_type}";
                 echo "mkdir $dirname";
                 if [ "$equation" = viscoelastic2 ]; then mechanisms=3; else mechanisms=0; fi;
                 mkdir -p $dirname && cd $dirname ;
                 pwd; 
                 CMAKE_PREFIX_PATH=~ ;
                 cmake -DNETCDF=OFF -DMETIS=ON -DCOMMTHREAD=ON -DASAGI=OFF -DHDF5=ON -DCMAKE_BUILD_TYPE=$build_type -DTESTING=ON -DLOG_LEVEL=warning -DLOG_LEVEL_MASTER=info -DARCH=hsw -DPRECISION=$precision -DEQUATIONS=$equation -DNUMBER_OF_MECHANISMS=$mechanisms -DGEMM_TOOLS_LIST=LIBXSMM .. ;
                 make -j $(nproc);
                 cd .. ; 
             done; 
          done;
          
    artifacts:
        paths:
            - build_*
            
ci_builder_test:
        stage: test
        tags:
            - ci_testing_cc
        allow_failure: true
        needs:
            - job: ci_builder_build
        parallel:
            matrix:
                - DIR: build_elastic_double_Release
                - DIR: build_elastic_single_Release
        script: 
            - echo "testing"
            - pwd
            - ls
            - cd $DIR
            - make test
            
gpu_pre_build:
    stage: pre_build
    tags:
        - atsccs68-docker-executor
    allow_failure: true
    variables:
        GIT_STRATEGY: clone
    image:
        name: ravilmobile/seissol-base:gcc-8_cuda-10
        entrypoint: [""]
    before_script:
        - git branch -vva
        - echo $commit_author_name
    script:
        - echo "HOST arch.:" $HOST
        - echo "GPU model:" $GPU_MODEL
        - nvidia-smi

# gcc@9 is required to compile and run hipSYCL@0.9.1. It is based on llvm and emits nvptx code by itself.
# gcc@8 is required for cuda@10.2
# Note: currently we cannot switch to cuda@11 because an old version of the nvidia driver is installed
#       on our GPU CI runner (atsccs68-docker-executor). Once it is updated we will be able
#       to switch to cuda@11 and thus to gcc@9
.common_gpu_test_script: &common_gpu_steps
    - export CTEST_OUTPUT_ON_FAILURE=1
    - if [[ "${BACKEND}" == "hipsycl" ]]; then
        export CC=/usr/bin/gcc-9 ;
        export CXX=/usr/bin/g++-9 ;
        export FC=/usr/bin/gfortran-9 ;
        export LIBRARY_PATH=/usr/local/cuda/lib64:$LIBRARY_PATH ;
        export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH ;
      else
        export CC=/usr/bin/gcc-8 ;
        export CXX=/usr/bin/g++-8 ;
      fi ;
    - if [[ "${BACKEND}" == "hip" ]]; then
        . /etc/profile.d/rocm.sh ;
        export HIP_PLATFORM=nvidia ;
      fi ;


gpu_build:
    stage: build
    tags:
        -  atsccs68-docker-executor
    allow_failure: false
    needs:
        - job: gpu_pre_build
    image:
        name: ravilmobile/seissol-base:gcc-8_cuda-10
        entrypoint: [""]
    parallel:
        matrix:
        - BACKEND: [cuda, hipsycl, hip]
    before_script:
        - git submodule init
        - sed -i 's/\.\.\/\.\./https\:\/\/github\.com/g' .gitmodules
        - git submodule sync
        - git submodule --quiet update --init --recursive 
    script:
        - *common_gpu_steps
        - pip3 install git+https://github.com/ravil-mobile/gemmforge.git@v0.0.206
        - set -euo pipefail
        - for precision in double single; do
            mkdir -p ./build_${BACKEND}_${precision} && cd ./build_${BACKEND}_${precision} ;
            cmake ..
            -DCMAKE_BUILD_TYPE=Release
            -DDEVICE_BACKEND=${BACKEND}
            -DDEVICE_ARCH=${GPU_MODEL}
            -DHOST_ARCH=${HOST}
            -DPRECISION=${precision}
            -DPROXY_PYBINDING=ON ;
            make -j 2 ;
            cd .. ;
          done;
        - set +u
    artifacts:
        paths:
            - build_*


gpu_convergence_test:
    stage: test
    tags:
        -  atsccs68-docker-executor
    allow_failure: false
    needs:
        - job: gpu_build
    image:
        name: ravilmobile/seissol-base:gcc-8_cuda-10
        entrypoint: [""]
    parallel:
        matrix:
        - BACKEND: [cuda, hipsycl, hip]
    script:
        - *common_gpu_steps
        - git clone https://github.com/SeisSol/Examples.git tests
        - pip3 install -r ./tests/convergence_elastic/requirements.txt
        - set -euo pipefail
        - for precision in double single; do
            cd ./build_${BACKEND}_${precision} ;
            echo "[${BACKEND}] Elastic Convergence test with precision - ${precision}" ;
            cp -r ../tests/convergence_elastic/* . ;
            PYTHONPATH=$PWD python3 ./elastic_convergence_runner
            --executable $PWD/SeisSol_Release_*
            --tmp-dir /tmp/seissol
            --sizes 4 8 16
            --expected-errors 1e-2 1e-4 5e-5
            --norm-type LInf
            --end-time 0.5
            --allow-run-as-root ;
            cd .. ;
          done;
        - set +u
    artifacts:
        paths:
            - build_*


gpu_performance_test:
    stage: performance_eval
    tags:
        -  atsccs68-docker-executor
    allow_failure: true
    needs:
        - job: gpu_convergence_test
    image:
        name: ravilmobile/seissol-base:gcc-8_cuda-10
        entrypoint: [""]
    parallel:
        matrix:
        - BACKEND: [cuda, hipsycl, hip]
    script:
        - *common_gpu_steps
        - set -euo pipefail
        - for precision in double single; do
            echo "[${BACKEND}] Running SeiSol proxy with precision - ${precision}" ;
            cd ./build_${BACKEND}_${precision} ;
            pip3 install -r ./proxy-runners/requirements.txt ;
            python3 ./proxy-runners/run-all.py -c 25000 -t 100 ;
            cat ./proxy_*.pd ;
            cd .. ;
          done;
        - set +u