build-and-test-linux.yml

name: Build and Test (Linux)

permissions:
  contents: write

on:
  push:
    branches:
      - main
  pull_request:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # Filter to allow skipping the test if no relevant changes occurred.
  filter:
    runs-on: ubuntu-latest
    outputs:
      test: ${{ steps.filter.outputs.test }}
    steps:
      - uses: actions/checkout@v6
      - uses: dorny/paths-filter@v4
        id: filter
        with:
          filters: |
            test:
              - 'palace/**'
              - 'cmake/**'
              - 'extern/**'
              - 'examples/**'
              - 'test/unit/**'
              - 'test/data/**'
              - '.github/workflows/build-and-test-linux.yml'

  build-and-test-linux:
    needs: filter
    strategy:
      fail-fast: false
      matrix:
        include:
        # x86 Intel stack.
        - arch: x86
          compiler: intel
          mpi: intelmpi
          math-libs: intelmkl
          build-shared: shared
          with-64bit-int: int64
          with-openmp: openmp
          with-eigensolver: slepc
          with-solver: superlu
        # x86 GCC / OpenBLAS open stack.
        - arch: x86
          compiler: gcc
          mpi: openmpi
          math-libs: openblas
          build-shared: shared
          with-64bit-int: int64
          with-openmp: openmp
          with-eigensolver: slepc
          with-solver: strumpack
        # arm GCC / OpenBLAS and ARPACK.
        - arch: arm
          compiler: gcc
          mpi: openmpi
          math-libs: openblas
          build-shared: static
          with-64bit-int: int64
          with-openmp: serial
          with-eigensolver: arpack
          with-solver: mumps
        # Sanitizer build (ASan/UBSan), x86 Clang.
        - arch: x86
          compiler: clang
          mpi: openmpi
          math-libs: openblas
          build-shared: shared
          with-64bit-int: int32
          with-openmp: serial
          with-eigensolver: slepc
          with-solver: superlu
          with-sanitizers: true
        # Coverage build, x86 GCC.
        - arch: x86
          compiler: gcc
          mpi: openmpi
          math-libs: openblas
          build-shared: shared
          with-64bit-int: int64
          with-openmp: openmp
          with-eigensolver: slepc
          with-solver: strumpack
          with-coverage: true

    # Flaky arm builds with ubuntu 24 means we pin to ubuntu 22
    # (See comment above matrix)
    runs-on: ${{ matrix.arch == 'x86' && 'palace_ubuntu-latest_16-core' || 'ubuntu-22.04-arm' }}
    steps:
      - uses: actions/checkout@v6
        if: needs.filter.outputs.test == 'true'
        with:
          submodules: 'recursive'
      - name: Update package lists
        if: needs.filter.outputs.test == 'true'
        run: |
          sudo apt-get update
      - name: Configure GNU compilers
        if: needs.filter.outputs.test == 'true'
        run: |
          # Find latest gnu compiler versions
          LATEST_GFORTRAN=$(apt-cache search '^gfortran-[0-9]+$' | sort -V | tail -n1 | cut -d' ' -f1)
          LATEST_GCC=$(apt-cache search '^gcc-[0-9]+$' | sort -V | tail -n1 | cut -d' ' -f1)
          LATEST_GPP=$(apt-cache search '^g\+\+-[0-9]+$' | sort -V | tail -n1 | cut -d' ' -f1)
          GFORT_VERSION=$(echo $LATEST_GFORTRAN | cut -d'-' -f2)
          GCC_VERSION=$(echo $LATEST_GCC | cut -d'-' -f2)
          sudo apt-get install -y $LATEST_GFORTRAN $LATEST_GCC $LATEST_GPP
          sudo update-alternatives --install $(which gfortran) gfortran $(which gfortran-$GFORT_VERSION) ${GFORT_VERSION}0
          sudo update-alternatives --set gfortran $(which gfortran-$GFORT_VERSION)
          sudo update-alternatives --install $(which gcc) gcc $(which gcc-$GCC_VERSION) ${GCC_VERSION}0 \
              --slave $(which g++) g++ $(which g++-$GCC_VERSION) \
              --slave $(which gcov) gcov $(which gcov-$GCC_VERSION) \
              --slave $(which gcov-tool) gcov-tool $(which gcov-tool-$GCC_VERSION)
          sudo update-alternatives --set gcc $(which gcc-$GCC_VERSION)

          gfortran --version
          gcc --version
          g++ --version

      - name: Configure LCOV
        if: needs.filter.outputs.test == 'true' && matrix.with-coverage
        run: |
          sudo apt-get install -y lcov

      - name: Configure Open MPI
        if: needs.filter.outputs.test == 'true' && matrix.mpi == 'openmpi'
        run: |
          sudo apt-get install -y openmpi-bin libopenmpi-dev
      - name: Configure MPICH
        if: needs.filter.outputs.test == 'true' && matrix.mpi == 'mpich'
        run: |
          sudo apt-get install -y mpich libmpich-dev

      - name: Configure Intel MPI
        if: needs.filter.outputs.test == 'true' && matrix.mpi == 'intelmpi'
        uses: mpi4py/setup-mpi@v1
        with:
          mpi: ${{ matrix.mpi }}

      - name: Configure Clang compiler
        if: needs.filter.outputs.test == 'true' && matrix.compiler == 'clang'
        run: |
          sudo apt-get install -y clang lld
          if [[ "${{ matrix.with-openmp }}" == 'openmp' ]]; then
            sudo apt-get install -y libomp-dev
          fi

      - name: Configure Intel oneAPI compiler
        if: needs.filter.outputs.test == 'true' && matrix.compiler == 'intel'
        run: |
          sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp \
                                  intel-oneapi-compiler-fortran

      - name: Install math libraries (OpenBLAS)
        if: needs.filter.outputs.test == 'true' && matrix.math-libs == 'openblas'
        run: |
          if [[ "${{ matrix.with-openmp }}" == 'openmp' ]]; then
            sudo apt-get install -y libopenblas-openmp-dev
          else
            sudo apt-get install -y libopenblas-serial-dev
          fi
      - name: Install GCC OpenMP runtime
        if: needs.filter.outputs.test == 'true' && matrix.with-openmp == 'openmp'
        run: |
          sudo apt-get install -y libgomp1

      - name: Install math libraries (Intel oneAPI MKL)
        if: needs.filter.outputs.test == 'true' && matrix.math-libs == 'intelmkl'
        run: |
          sudo apt-get install -y intel-oneapi-mkl intel-oneapi-mkl-devel

      - name: Install math libraries (AOCL)
        if: needs.filter.outputs.test == 'true' && matrix.math-libs == 'aocl'
        run: |
          wget -q https://download.amd.com/developer/eula/aocl/aocl-4-1/aocl-linux-gcc-4.1.0_1_amd64.deb
          sudo apt-get install -y ./aocl-linux-gcc-4.1.0_1_amd64.deb
          rm aocl-linux-gcc-*.deb
          echo "AOCLROOT=/opt/AMD/aocl/aocl-linux-gcc-4.1.0/gcc" >> $GITHUB_ENV

      - name: Install math libraries (ARMPL)
        if: needs.filter.outputs.test == 'true' && matrix.math-libs == 'armpl'
        run: |
          wget -q https://developer.arm.com/-/cdn-downloads/permalink/Arm-Performance-Libraries/Version_24.10/arm-performance-libraries_24.10_deb_gcc.tar
          tar -xf arm-performance-libraries* && rm -rf arm-performance-libraries*.tar
          sudo ./arm-performance-libraries*/arm-performance-libraries*.sh -a -f -i /opt/arm
          echo "ARMPL_DIR=/opt/arm/armpl_24.10_gcc" >> $GITHUB_ENV

      - name: Test MPI installation
        if: needs.filter.outputs.test == 'true'
        run: |
          # Create test program
          cat > mpi_test.c << 'EOF'
          #include <stdio.h>
          #include <mpi.h>

          int main(int argc, char *argv[]) {
              int rank, size;

              MPI_Init(&argc, &argv);
              MPI_Comm_rank(MPI_COMM_WORLD, &rank);
              MPI_Comm_size(MPI_COMM_WORLD, &size);

              printf("Process %d of %d\n", rank, size);

              if (rank == 0) {
                  if (size == 1) {
                      printf("ERROR: MPI_Comm_size reports only 1 process\n");
                      return 1;
                  } else {
                      printf("SUCCESS: MPI running with %d processes\n", size);
                  }
              }

              MPI_Finalize();
              return 0;
          }
          EOF

          # Select correct compiler wrapper
          if [[ "${{ matrix.mpi }}" == "mpich" ]]; then
              MPICC="mpicc"
          elif [[ "${{ matrix.mpi }}" == "openmpi" ]]; then
              MPICC="mpicc"
          elif [[ "${{ matrix.mpi }}" == "intelmpi" ]]; then
              source /opt/intel/oneapi/setvars.sh
              MPICC="mpiicx"
          fi

          $MPICC mpi_test.c -o mpi_test
          mpirun -np 2 ./mpi_test

      # Julia is needed for the antenna postprocessing example below; the
      # regression sweep itself uses the Catch2 binary and doesn't need it.
      - uses: julia-actions/setup-julia@v2
        if: needs.filter.outputs.test == 'true'
        with:
          version: '1'

      - uses: julia-actions/cache@v3
        if: needs.filter.outputs.test == 'true'

      - name: Build Palace
        if: needs.filter.outputs.test == 'true'
        env:
          CMAKE_BUILD_TYPE: Release
          NUM_PROC_BUILD_MAX: '32'
          # Catch2 runs the binary as part of the cmake test discovery step. So,
          # we want to avoid reporting leaks.
          ASAN_OPTIONS: detect_leaks=0
        run: |
          # Configure environment
          if [[ "${{ matrix.compiler }}" == 'intel' ]] || \
             [[ "${{ matrix.mpi }}" == 'intelmpi' ]] || \
             [[ "${{ matrix.math-libs }}" == 'intelmkl' ]]; then
            source /opt/intel/oneapi/setvars.sh  # Sets PATH, MKLROOT
            if [[ "${{ matrix.compiler }}" == 'intel' ]]; then
              export CC=icx
              export CXX=icpx
              export FC=ifx
            fi
          elif [[ "${{ matrix.compiler }}" == 'clang' ]]; then
            export CC=clang
            export CXX=clang++
            export FC=gfortran
            export LDFLAGS='-fuse-ld=lld'
          elif [[ "${{ matrix.compiler }}" == 'gcc' ]]; then
            export CC=gcc
            export CXX=g++
            export FC=gfortran
          fi
          export NUM_PROC_BUILD=$(nproc 2> /dev/null || sysctl -n hw.ncpu)
          if [[ "$NUM_PROC_BUILD" -gt "$NUM_PROC_BUILD_MAX" ]]; then
            NUM_PROC_BUILD=$NUM_PROC_BUILD_MAX
          fi
          if [[ "${{ matrix.math-libs }}" == 'aocl' ]]; then
            export LD_LIBRARY_PATH="$AOCLROOT/lib:$LD_LIBRARY_PATH"
          fi
          if [[ "${{ matrix.math-libs }}" == 'armpl' ]]; then
            export LD_LIBRARY_PATH="$ARMPL_DIR/lib:$LD_LIBRARY_PATH"
          fi

          [[ "${{ matrix.build-shared }}" == 'shared' ]] && BUILD_SHARED='ON' || BUILD_SHARED='OFF'
          [[ "${{ matrix.with-64bit-int }}" == 'int64' ]] && WITH_INT64='ON' || WITH_INT64='OFF'
          [[ "${{ matrix.with-openmp }}" == 'openmp' ]] && WITH_OPENMP='ON' || WITH_OPENMP='OFF'

          [[ "${{ matrix.with-solver }}" == 'superlu' ]] && WITH_SUPERLU='ON' || WITH_SUPERLU='OFF'
          [[ "${{ matrix.with-solver }}" == 'strumpack' ]] && WITH_STRUMPACK='ON' || WITH_STRUMPACK='OFF'
          [[ "${{ matrix.with-solver }}" == 'mumps' ]] && WITH_MUMPS='ON' || WITH_MUMPS='OFF'
          [[ "${{ matrix.with-eigensolver }}" == 'slepc' ]] && WITH_SLEPC='ON' || WITH_SLEPC='OFF'
          [[ "${{ matrix.with-eigensolver }}" == 'arpack' ]] && WITH_ARPACK='ON' || WITH_ARPACK='OFF'
          [[ "${{ matrix.with-sanitizers }}" == 'true' ]] && WITH_SANITIZERS='ON' || WITH_SANITIZERS='OFF'
          [[ "${{ matrix.with-coverage }}" == 'true' ]] && WITH_COVERAGE='ON' || WITH_COVERAGE='OFF'
          [[ "${{ matrix.with-butterflypack }}" == 'true' ]] && WITH_BUTTERFLYPACK='ON' || WITH_BUTTERFLYPACK='OFF'


          # Build and install (with unit tests)
          # Compile so that MFEM throws exceptions rather then aborts for unit-tests
          # Unit [Parallel] cases run at PALACE_TESTS_NUMPROC ranks (small, fast);
          # regression cases run at PALACE_REGRESSION_NUMPROC (~8 CPUs/case --
          # fewer ranks shift tail eigenvalues past tolerance for some solvers).
          # CTest derives PROCESSORS = ranks x threads and pins OMP_NUM_THREADS,
          # so -j only bounds concurrency.
          TEST_CORES=$(nproc 2> /dev/null || sysctl -n hw.ncpu)
          if [[ "$TEST_CORES" -gt 8 ]]; then
            TEST_CORES=8
          fi
          if [[ "${{ matrix.with-openmp }}" == 'openmp' ]]; then
            TESTS_OMP_THREADS=2
          else
            TESTS_OMP_THREADS=1
          fi
          REGRESSION_NUMPROC=$(( TEST_CORES / TESTS_OMP_THREADS ))
          if [[ "$REGRESSION_NUMPROC" -lt 1 ]]; then
            REGRESSION_NUMPROC=1
          fi
          mkdir palace-build && cd palace-build
          cmake .. \
            -DCMAKE_INSTALL_PREFIX=$(pwd)/../palace-install \
            -DPALACE_TESTS_NUMPROC=2 \
            -DPALACE_REGRESSION_NUMPROC=$REGRESSION_NUMPROC \
            -DPALACE_TESTS_OMP_THREADS=$TESTS_OMP_THREADS \
            -DBUILD_SHARED_LIBS=$BUILD_SHARED \
            -DPALACE_WITH_64BIT_INT=$WITH_INT64 \
            -DPALACE_WITH_OPENMP=$WITH_OPENMP \
            -DPALACE_WITH_SUPERLU=$WITH_SUPERLU \
            -DPALACE_WITH_STRUMPACK=$WITH_STRUMPACK \
            -DPALACE_WITH_STRUMPACK_BUTTERFLYPACK=$WITH_BUTTERFLYPACK \
            -DPALACE_WITH_MUMPS=$WITH_MUMPS \
            -DPALACE_WITH_SLEPC=$WITH_SLEPC \
            -DPALACE_WITH_ARPACK=$WITH_ARPACK \
            -DPALACE_MFEM_USE_EXCEPTIONS=ON \
            -DPALACE_BUILD_WITH_SANITIZERS=$WITH_SANITIZERS \
            -DPALACE_BUILD_WITH_COVERAGE=$WITH_COVERAGE \
            -DPALACE_MFEM_USE_EXCEPTIONS=ON

          make -j$NUM_PROC_BUILD palace-tests

      - name: Run unit tests
        if: needs.filter.outputs.test == 'true'
        timeout-minutes: 10
        run: |
          # Configure environment
          if [[ "${{ matrix.compiler }}" == 'intel' ]] || \
             [[ "${{ matrix.mpi }}" == 'intelmpi' ]] || \
             [[ "${{ matrix.math-libs }}" == 'intelmkl' ]]; then
            source /opt/intel/oneapi/setvars.sh  # Sets PATH, MKLROOT
          fi
          export LD_LIBRARY_PATH=$(pwd)/palace-install/lib:$LD_LIBRARY_PATH
          export LD_LIBRARY_PATH=$(pwd)/palace-install/lib64:$LD_LIBRARY_PATH
          if [[ "${{ matrix.math-libs }}" == 'aocl' ]]; then
            export LD_LIBRARY_PATH="$AOCLROOT/lib:$LD_LIBRARY_PATH"
          fi
          if [[ "${{ matrix.math-libs }}" == 'armpl' ]]; then
            export LD_LIBRARY_PATH="$ARMPL_DIR/lib:$LD_LIBRARY_PATH"
          fi

          # Configure sanitizer options
          if [[ "${{ matrix.with-sanitizers }}" == 'true' ]]; then
            export ASAN_OPTIONS=detect_leaks=1
            export UBSAN_OPTIONS=print_stacktrace=1
            # Suppress known leaks in third-party MPI/hwloc/libCEED libraries.
            # See test/unit/lsan_suppressions.txt for the suppression list.
            export LSAN_OPTIONS="suppressions=$GITHUB_WORKSPACE/test/unit/lsan_suppressions.txt"
          fi

          # We don't compute coverage for the entire matrix because it slows
          # everything down.
          if [[ "${{ matrix.with-coverage }}" == 'true' ]]; then
            scripts/measure-test-coverage report $(pwd)/palace-build
          else
            cd $(pwd)/palace-build/palace-build/test/unit
            # Skip [Regression] and [Long] cases here; they have their
            # own slower steps further down. CTest pins OMP_NUM_THREADS and
            # reserves PROCESSORS per case, so -j just bounds concurrency.
            ctest -j "$(nproc 2> /dev/null || sysctl -n hw.ncpu)" --output-on-failure -LE "^(regression|long)$"
          fi

      # NOTE: Fork PRs get a read-only GITHUB_TOKEN and cannot push to gh-pages;
      # for those the coverage report is still available via the artifact below.
      - name: Deploy coverage to gh-pages
        if: |
          needs.filter.outputs.test == 'true' && matrix.with-coverage &&
          (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
        uses: peaceiris/actions-gh-pages@v4
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: palace-build/coverage_html
          destination_dir: ${{ github.event_name == 'pull_request' && format('previews/PR{0}/coverage', github.event.pull_request.number) || format('coverage/{0}', github.sha) }}
          keep_files: true

      - name: Add coverage link to summary
        if: |
          needs.filter.outputs.test == 'true' && matrix.with-coverage &&
          (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
        env:
          EVENT_NAME: ${{ github.event_name }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
          SHA: ${{ github.sha }}
        run: |
          if [[ "$EVENT_NAME" == "pull_request" ]]; then
            COVERAGE_URL="https://awslabs.github.io/palace/previews/PR${PR_NUMBER}/coverage/"
          else
            COVERAGE_URL="https://awslabs.github.io/palace/coverage/${SHA}/"
          fi
          echo "### Coverage Report" >> "$GITHUB_STEP_SUMMARY"
          echo "View the coverage report at: [Coverage Report](${COVERAGE_URL})" >> "$GITHUB_STEP_SUMMARY"

      - name: Upload coverage report
        if: needs.filter.outputs.test == 'true' && matrix.with-coverage
        uses: actions/upload-artifact@v7
        with:
          name: coverage-${{ matrix.arch }}-${{ matrix.compiler }}-${{ matrix.mpi }}
          path: palace-build/coverage_html/

      - name: Run regression tests
        if: needs.filter.outputs.test == 'true' && !matrix.with-sanitizers
        # CTest owns per-case scheduling for the [Regression] cases (label
        # "regression"). The configured MPI rank count and PROCESSORS property
        # reserve matching ctest slots so -j can fan out safely.
        timeout-minutes: 60
        run: |
          # Configure environment
          if [[ "${{ matrix.compiler }}" == 'intel' ]] || \
             [[ "${{ matrix.mpi }}" == 'intelmpi' ]] || \
             [[ "${{ matrix.math-libs }}" == 'intelmkl' ]]; then
            source /opt/intel/oneapi/setvars.sh  # Sets PATH, MKLROOT
          fi
          if [[ "${{ matrix.math-libs }}" == 'aocl' ]]; then
            export LD_LIBRARY_PATH="$AOCLROOT/lib:$LD_LIBRARY_PATH"
          fi
          if [[ "${{ matrix.math-libs }}" == 'armpl' ]]; then
            export LD_LIBRARY_PATH="$ARMPL_DIR/lib:$LD_LIBRARY_PATH"
          fi
          export PATH=$(pwd)/palace-install/bin:$PATH

          cd palace-build/palace-build/test/unit
          # CTest pins OMP_NUM_THREADS and reserves PROCESSORS per case, so -j is
          # just an upper bound on concurrency and can safely be all cores.
          ctest -j "$(nproc 2> /dev/null || sysctl -n hw.ncpu)" --output-on-failure -L "^regression$"

      - name: Check antenna postprocessing example
        # The Julia plot script that ships with the antenna example is
        # exercised here so it doesn't bit-rot. Gated to one matrix entry
        # to avoid paying the ~7 min Julia precompile per job. Reads the
        # farfield CSV the regression step left in the staging tree.
        if: needs.filter.outputs.test == 'true' && matrix.arch == 'x86' && matrix.compiler == 'gcc' && matrix.math-libs == 'openblas' && !matrix.with-coverage
        run: |
          STAGE_ROOT="${TMPDIR:-/tmp}/palace-regression"
          FARFIELD="$STAGE_ROOT/antenna/antenna_short_dipole/postpro/antenna_short_dipole/farfield-rE.csv"
          if [[ ! -f "$FARFIELD" ]]; then
            echo "Expected regression output missing: $FARFIELD" >&2
            exit 1
          fi
          julia --project=examples -e 'using Pkg; Pkg.instantiate()'
          julia --project=examples examples/antenna/plot_farfield.jl --model short_dipole --file "$FARFIELD"