action.yml

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

name: 'Palace CI'
description: 'Run Palace CI with Spack'
inputs:
  toolchain:
    description: 'Toolchain to use'
    required: true
  variant:
    description: 'Palace variant'
    required: false
    default: ''
  math-libs:
    description: 'Math libraries'
    required: false
    default: ''
  run-regression-tests:
    description: 'Run regression tests after build'
    required: false
    default: 'true'
  use-develop-deps:
    description: 'Use develop versions of key dependencies'
    required: false
    default: 'false'
  test-cases:
    description: 'Test cases: "default", "long", or space-separated list'
    required: false
    default: 'default'
  additional-test-cases:
    description: 'Additional test cases to run after default tests (space-separated list)'
    required: false
    default: ''

runs:
  using: 'composite'
  steps:
    - name: Setup environment
      uses: ./.github/actions/setup-runner
      with:
        julia-version: 'release'
        # Spack develop because we want to have access to the most recent
        # version of spack-packages
        spack-version: 'develop'
        # LLVM 19 because it is the latest supported by CUDA 12.9
        llvm-version: ${{ inputs.toolchain == 'llvm' && '19' || 'dont-install' }}
        setup-intel: ${{ inputs.toolchain == 'intel-oneapi' && 'true' || 'false' }}

    - name: Prepare spack.yaml
      shell: bash
      run: |
        source /tmp/timing_functions.sh
        start_timer
        if [[ "${{ inputs.variant }}" == *"+cuda"* ]]; then
          CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
          CUDA_ARGS="cuda_arch=${CUDA_ARCH}"
        else
          CUDA_ARGS=""
        fi

        # As of version 1.2.0-alpha, the spack concretizer hangs when using flang.
        # Also https://github.com/libxsmm/libxsmm/issues/996
        # TODO: Try again in the future to see if we can use it.
        if [[ "${{ inputs.toolchain }}" == "llvm" ]]; then
          FORTRAN_COMPILER="gcc"
        else
          FORTRAN_COMPILER="${{ inputs.toolchain }}"
        fi

        if [[ "${{ inputs.toolchain }}" == "intel-oneapi" ]]; then
          MATH_LIBS="${{ inputs.math-libs || 'intel-oneapi-mkl' }}"
          MPI_IMPL="intel-oneapi-mpi"
          C_CXX_COMPILER="intel-oneapi-compilers"
          FORTRAN_COMPILER="intel-oneapi-compilers"
        else
          # OpenBlas 0.3.30 has issues on arm
          # https://github.com/OpenMathLib/OpenBLAS/issues/5459
          MATH_LIBS="${{ inputs.math-libs || 'openblas@0.3.29' }}"
          MPI_IMPL="openmpi"
          C_CXX_COMPILER="${{ inputs.toolchain }}"
        fi
        PALACE_SPEC="local.palace@develop+libxsmm+superlu-dist+mumps+sundials+strumpack+slepc+arpack${{ inputs.variant }} ${CUDA_ARGS}"

        cat << EOF > spack.yaml
        spack:
          specs:
            - ${PALACE_SPEC}
          view: false
          config:
            install_tree:
              root: ${HOME}/opt/spack
              # We add a lot of padding to support better cache relocation
              # https://spack.readthedocs.io/en/latest/binary_caches.html#relocation
              padded_length: 256
          concretizer:
            reuse: false
            unify: true
            # generic cpu target so that we can increase cache hits
            # (we also do not care too much about hyper-optimal performance)
            targets:
              granularity: generic
          packages:
            petsc:
              require: ~hdf5
            mpi:
              require: ${MPI_IMPL}
            blas:
              require: ${MATH_LIBS}
            lapack:
              require: ${MATH_LIBS}
            c:
              require: [${C_CXX_COMPILER}]
            cxx:
              require: [${C_CXX_COMPILER}]
            fortran:
              require: [${FORTRAN_COMPILER}]
            # Metis is verify fragile
            # -fPIC seems to be required because metis always compiles some shared objects
            # -lc seems to be required for intel compilers
            metis:
              require: cflags=-fPIC cppflags=-fPIC ldflags=-lc
            parmetis:
              require: ldflags=-lc
        EOF

        # The Intel compiler has problems with C/C++/Fortran interop in MUMPS.
        # Fixes: for_main.c:(.text+0x19): undefined reference to `MAIN__'
        if [[ "${{ inputs.toolchain }}" == "intel-oneapi" ]]; then
          cat << INTELEOF >> spack.yaml
            mumps:
              require: fflags=-nofor-main
        INTELEOF
        fi

        if [[ "${{ inputs.use-develop-deps }}" == "true" ]]; then
          cat << DEVEOF >> spack.yaml
            hypre:
              require: "@develop"
            mfem:
              require: "@develop"
        DEVEOF
        fi

        if [[ "${{ inputs.variant }}" == *"+cuda"* ]]; then
          cat << CUDAEOF >> spack.yaml
            cuda:
              buildable: false
              externals:
              - spec: cuda@12.9
                prefix: /usr/local/cuda-12.9/
            # When some packages are built with OpenMP in a non-OpenMP build,
            # FindSTRUMPACK in MFEM cannot compile the test program. Hopefully,
            # we can remove this when MFEM is not in the superbuild.
            all:
              require: ~openmp
        CUDAEOF
        fi

        cat << EOF >> spack.yaml
          repos:
          - spack_repo/local
          mirrors:
            spack:
              binary: true
              url: https://binaries.spack.io/develop
            local-buildcache:
              binary: true
              url: oci://ghcr.io/awslabs/palace-develop-testing
              signed: false
              access_pair:
                id_variable: GITHUB_USER
                secret_variable: GITHUB_TOKEN
        EOF

    - name: Configure Compilers
      shell: bash
      run: |
        source /tmp/timing_functions.sh
        start_timer
        if [[ "${{ inputs.toolchain }}" == "intel-oneapi" ]]; then
          . /opt/intel/oneapi/setvars.sh
        fi
        spack -e . compiler find && spack -e . compiler list
        end_timer "Configure Compilers"

    # Spack and oneAPI-2025.3.1 have some compatibility problems where the
    # version of the fortran compiler is not correctly identified. As a
    # workaround, we manually edit the relevant entry in the spack.yaml to put
    # all the compilers under the same version. # This amounts to merging two
    # entries (for two different versions) into a single one.
    #
    # TODO: Check if this issue is still relevant, if not, remove this step.
    - name: Merge Intel Compiler Entries
      if: inputs.toolchain == 'intel-oneapi'
      shell: bash
      run: |
        python3 << 'EOF'
        import yaml
        
        with open('spack.yaml', 'r') as f:
            data = yaml.safe_load(f)
        
        # Get Intel compiler externals
        intel_externals = data['spack']['packages']['intel-oneapi-compilers']['externals']
        
        if len(intel_externals) > 1:
            # Merge compilers from all entries
            merged_compilers = {}
            newest_spec = None
            newest_prefix = None
            
            for entry in intel_externals:
                # Keep track of the newest version
                if newest_spec is None or entry['spec'] > newest_spec:
                    newest_spec = entry['spec']
                    newest_prefix = entry['prefix']
                
                # Merge all compilers
                if 'extra_attributes' in entry and 'compilers' in entry['extra_attributes']:
                    compilers = entry['extra_attributes']['compilers']
                    merged_compilers.update(compilers)
            
            # Create single merged entry
            merged_entry = {
                'spec': newest_spec,
                'prefix': newest_prefix,
                'extra_attributes': {
                    'compilers': merged_compilers
                }
            }
            
            # Replace ALL entries with single merged entry
            data['spack']['packages']['intel-oneapi-compilers']['externals'] = [merged_entry]
            
            # Write back
            with open('spack.yaml', 'w') as f:
                yaml.dump(data, f, default_flow_style=False)
            
            print("Merged Intel compiler entries into single entry")
        EOF

    - name: Display spack.yaml
      shell: bash
      run: |
        echo "=== Generated spack.yaml ==="
        cat spack.yaml
        echo "=== End spack.yaml ==="

    - name: Configure Binary Mirror Keys
      shell: bash
      run: |
        source /tmp/timing_functions.sh
        start_timer
        spack -e . buildcache keys --install --trust
        end_timer "Configure Binary Mirror Keys"

    - name: Bootstrap
      shell: bash
      run: |
        source /tmp/timing_functions.sh
        start_timer
        spack -e . bootstrap now
        end_timer "Bootstrap"

    - name: Concretize
      shell: bash
      env:
        PALACE_REF: ${{ github.head_ref || github.ref_name }}
      run: |
        source /tmp/timing_functions.sh
        start_timer
        spack -e . develop --path=$(pwd) local.palace@git."${PALACE_REF}"=develop
        spack -e . concretize -f --test root
        end_timer "Concretize"

    - name: Build Dependencies
      shell: bash
      run: |
        source /tmp/timing_functions.sh
        start_timer
        spack -e . install --only-concrete --no-check-signature --fail-fast --show-log-on-error --only dependencies --test root -j $(nproc)
        end_timer "Build Dependencies"

    - name: Build Palace
      shell: bash
      run: |
        source /tmp/timing_functions.sh
        start_timer
        spack -e . install --only-concrete --keep-stage --show-log-on-error --only package --no-cache --test root -j $(nproc)
        end_timer "Build Palace"

    # We don't want to run unit tests as part of the long test run. They are
    # already run elsewhere.
    - name: Run Unit Tests
      if: inputs.test-cases != 'long'
      shell: bash
      run: |
        source /tmp/timing_functions.sh
        start_timer
        eval "$(spack -e . load --sh palace)"

        # Use nproc, but halve to account for hyperthreading.
        # NOTE: This CI action only runs on Linux runners (lscpu/nproc are Linux-specific).
        LOGICAL_CORES=$(nproc)
        THREADS_PER_CORE=$(lscpu | grep "Thread(s) per core" | awk '{print $4}')
        if [ "$THREADS_PER_CORE" = "2" ]; then
            export NUM_PROC_TEST=$((LOGICAL_CORES / 2))
        else
            export NUM_PROC_TEST=$LOGICAL_CORES
        fi

        if [[ "${{ inputs.variant }}" == *"+openmp"* ]]; then
          NUM_PROC_TEST=$(( NUM_PROC_TEST / 2 ))
          export OMP_NUM_THREADS=2
        else
          export OMP_NUM_THREADS=1
        fi

        cd "$(spack -e . location -b palace)"

        # Skip serial/mpi libCEED tests on CUDA builds due to umpire host memory
        # allocation bug.
        #
        # TODO: Remove this when MFEM/Palace switches to using standard host
        # allocation.
        if [[ "${{ inputs.variant }}" == *"+cuda"* ]]; then
          CTEST_EXCLUDE="--exclude-regex (serial|mpi)-.*libCEED"
        else
          CTEST_EXCLUDE=""
        fi

        # Disable OpenMPI 5.x default CPU binding so MPI processes don't starve
        # OpenMP threads during parallel test execution. OpenMPI 5.x uses PRRTE,
        # so we set both the PRTE and legacy OMPI MCA variables for robustness.
        export PRTE_MCA_hwloc_default_binding_policy=none
        export OMPI_MCA_hwloc_base_binding_policy=none

        # Runs all hardware-compatible tests, skipping Catch2 benchmarks.
        ctest -j $NUM_PROC_TEST --output-on-failure $CTEST_EXCLUDE

        end_timer "Run Unit Tests"

    # We upload the spack environment as an artifact so that next runners can
    # pick up the very same environment and have consistent binaries (pulled
    # from the cache).
    - name: Upload spack environment
      uses: actions/upload-artifact@v4
      if: inputs.run-regression-tests != 'true'
      with:
        name: spack-env-${{ runner.arch }}-${{ inputs.toolchain }}-${{ inputs.variant || 'default' }}-${{ inputs.math-libs || 'default' }}
        path: |
          spack.yaml
          spack.lock
        retention-days: 7

    - name: Push to GHCR cache
      shell: bash
      run: |
        source /tmp/timing_functions.sh
        start_timer
        spack -e . buildcache push --force --with-build-dependencies --unsigned --update-index local-buildcache || true
        end_timer "Push to GHCR cache"

    - name: Run Regression Tests
      if: inputs.run-regression-tests == 'true'
      uses: ./.github/actions/run-regression-tests
      with:
        testing-on-build-runner: 'true'
        variant: ${{ inputs.variant }}
        test-cases: ${{ inputs.test-cases }}
        additional-test-cases: ${{ inputs.additional-test-cases }}

    # We print timing if we don't run the regression tests, otherwise, it will
    # be regression test job that prints the file. (This is for the build-xxx
    # jobs)
    - name: Timing information
      if: inputs.run-regression-tests != 'true'
      shell: bash
      run: |
        cat /tmp/timing.log