Reapply cuDSS support after merging origin/main into PR #717 branch

Author: hughcars

.github/license-check/c.txt

@@ -1,2 +1 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0

CMakeLists.txt

@@ -41,6 +41,10 @@ set(PALACE_WITH_GSLIB ON CACHE BOOL "Build with GSLIB library for high-order fie
 set(PALACE_WITH_STRUMPACK_BUTTERFLYPACK OFF CACHE BOOL "Build with ButterflyPACK support for STRUMPACK solver")
 set(PALACE_WITH_STRUMPACK_ZFP OFF CACHE BOOL "Build with ZFP support for STRUMPACK solver")
 
+set(CUDSS_DIR "" CACHE STRING
+  "Path to cuDSS installation directory (not required if already on CMAKE_PREFIX_PATH)"
+)
+
 set(PALACE_WITH_SUNDIALS ON CACHE BOOL "Build with SUNDIALS differential/algebraic equations solver")
 
 set(ANALYZE_SOURCES_CLANG_TIDY OFF CACHE BOOL "Run static analysis checks using clang-tidy")
@@ -58,6 +62,9 @@ endif()
 if(PALACE_WITH_CUDA AND PALACE_WITH_HIP)
   message(FATAL_ERROR "PALACE_WITH_CUDA is not compatible with PALACE_WITH_HIP")
 endif()
+if(PALACE_WITH_CUDSS AND NOT PALACE_WITH_CUDA)
+  message(FATAL_ERROR "PALACE_WITH_CUDSS requires PALACE_WITH_CUDA")
+endif()
 if(PALACE_WITH_CUDA)
   # Note: The new behavior of CMake policy CMP0104 will initialize CMAKE_CUDA_ARCHITECTURES
   # to an (old) compatible value even when not set by the user.
@@ -151,11 +158,6 @@ if(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_CXX_COMPILER_ID STREQUAL "Intel
 have been replaced by the newer Clang-based icx/icpx from Intel oneAPI")
 endif()
 
-# cuDSS is only for CUDA builds
-if(PALACE_WITH_CUDSS AND NOT PALACE_WITH_CUDA)
-  message(FATAL_ERROR "PALACE_WITH_CUDSS requires PALACE_WITH_CUDA")
-endif()
-
 # MAGMA is only for GPU builds
 if(PALACE_WITH_MAGMA AND NOT (PALACE_WITH_CUDA OR PALACE_WITH_HIP))
   message(STATUS "Disabling MAGMA due to lack of CUDA or HIP support")

cmake/ExternalGitTags.cmake

@@ -125,15 +125,15 @@ set(EXTERN_METIS_GIT_TAG
 
 # MFEM
 set(EXTERN_MFEM_URL
-  "https://github.com/yuyangdai/mfem.git" CACHE STRING
+  "https://github.com/mfem/mfem.git" CACHE STRING
   "URL for external MFEM build"
 )
 set(EXTERN_MFEM_GIT_BRANCH
-  "cudss-dev" CACHE STRING
+  "master" CACHE STRING
   "Git branch for external MFEM build"
 )
 set(EXTERN_MFEM_GIT_TAG
-  "e217864f168acb1d7bdb3fd7c5aa73ef81951237" CACHE STRING
+  "d9d6526cc1749980a2ba1da16e2c1ca1e07d82ec" CACHE STRING
   "Git tag for external MFEM build"
 )
 

cmake/ExternalMFEM.cmake

@@ -123,7 +123,7 @@ if(PALACE_WITH_CUDA)
     "-DCMAKE_CUDA_FLAGS=${MFEM_CUDA_FLAGS}"
     "-DMFEM_USE_CUDSS=${PALACE_WITH_CUDSS}"
   )
-  if(PALACE_WITH_CUDSS)
+  if(PALACE_WITH_CUDSS AND NOT "${CUDSS_DIR}" STREQUAL "")
     list(APPEND MFEM_OPTIONS
       "-DCUDSS_DIR=${CUDSS_DIR}"
     )
@@ -407,6 +407,7 @@ set(MFEM_PATCH_FILES
   "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_gmsh_parser_performance.diff"
   "${CMAKE_SOURCE_DIR}/extern/patch/mfem/mfem_pr5246.diff"
   "${CMAKE_SOURCE_DIR}/extern/patch/mfem/mfem_pr5353.diff"
+  "${CMAKE_SOURCE_DIR}/extern/patch/mfem/mfem_pr5124_cudss.diff"
 )
 
 include(ExternalProject)

cmake/ExternalPalace.cmake

@@ -45,6 +45,9 @@ if(NOT "${MUMPS_DIR}" STREQUAL "")
 elseif(PALACE_BUILD_EXTERNAL_DEPS AND PALACE_WITH_MUMPS)
   list(APPEND PALACE_OPTIONS "-DMUMPS_DIR=${CMAKE_INSTALL_PREFIX}")
 endif()
+if(PALACE_WITH_CUDSS AND NOT "${CUDSS_DIR}" STREQUAL "")
+  list(APPEND PALACE_OPTIONS "-DCUDSS_DIR=${CUDSS_DIR}")
+endif()
 if(NOT "${STRUMPACK_DIR}" STREQUAL "")
   list(APPEND PALACE_OPTIONS "-DSTRUMPACK_DIR=${STRUMPACK_DIR}")
 elseif(PALACE_BUILD_EXTERNAL_DEPS AND PALACE_WITH_STRUMPACK)
@@ -55,9 +58,6 @@ if(NOT "${SUPERLU_DIST_DIR}" STREQUAL "")
 elseif(PALACE_BUILD_EXTERNAL_DEPS AND PALACE_WITH_SUPERLU)
   list(APPEND PALACE_OPTIONS "-DSUPERLU_DIST_DIR=${CMAKE_INSTALL_PREFIX}")
 endif()
-if(PALACE_WITH_CUDSS AND NOT "${CUDSS_DIR}" STREQUAL "")
-  list(APPEND PALACE_OPTIONS "-DCUDSS_DIR=${CUDSS_DIR}")
-endif()
 if(NOT "${METIS_DIR}" STREQUAL "")
   list(APPEND PALACE_OPTIONS "-DMETIS_DIR=${METIS_DIR}")
 elseif(PALACE_BUILD_EXTERNAL_DEPS)

docs/src/config/solver.md

@@ -512,6 +512,9 @@ equations to be solved for each simulation type. The available options are:
     uses a real approximation to the true complex linear system matrix. This option is only
     available when *Palace* has been
     [built with MUMPS support](../install.md#Configuration-options).
+  - `"cuDSS"` :  The [NVIDIA cuDSS](https://developer.nvidia.com/cudss) GPU sparse
+    direct solver is used to factorize the system matrix. This option is only available
+    when *Palace* has been built with CUDA and cuDSS support.
   - `"AMS"` :  Hypre's
     [Auxiliary-space Maxwell Solver (AMS)](https://hypre.readthedocs.io/en/latest/solvers-ams.html),
     an algebraic multigrid (AMG)-based preconditioner.

docs/src/install.md

@@ -192,6 +192,8 @@ Additional build options are (with default values in brackets):
   - `PALACE_WITH_SUPERLU [ON]` :  Build with SuperLU_DIST sparse direct solver
   - `PALACE_WITH_STRUMPACK [OFF]` :  Build with STRUMPACK sparse direct solver
   - `PALACE_WITH_MUMPS [OFF]` :  Build with MUMPS sparse direct solver
+  - `PALACE_WITH_CUDSS [OFF]` :  Build with cuDSS sparse direct solver (requires
+    `PALACE_WITH_CUDA=ON` and a cuDSS installation, optionally specified with `CUDSS_DIR`)
   - `PALACE_WITH_SLEPC [ON]` :  Build with SLEPc eigenvalue solver
   - `PALACE_WITH_ARPACK [OFF]` :  Build with ARPACK eigenvalue solver
   - `PALACE_WITH_LIBXSMM [ON]` :  Build with LIBXSMM backend for libCEED
@@ -255,7 +257,8 @@ installation of [libCEED](https://github.com/CEED/libCEED), and
 
 As part of the [Build from source](#Build-from-source), the CMake build will automatically
 build and install a small number of third-party dependencies before building *Palace*. The
-source code for these dependencies is downloaded during the build process:
+source code for these dependencies is downloaded during the build process (except for
+binary/user-provided packages such as cuDSS):
 
   - [METIS](http://glaros.dtc.umn.edu/gkhome/metis/metis/overview) and
     [ParMETIS](http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview)
@@ -267,6 +270,7 @@ source code for these dependencies is downloaded during the build process:
     [ButterflyPACK](https://github.com/liuyangzhuan/ButterflyPACK) and
     [zfp](https://github.com/LLNL/zfp) support
   - [MUMPS](http://mumps.enseeiht.fr/) (optional, when `PALACE_WITH_MUMPS=ON`)
+  - [cuDSS](https://developer.nvidia.com/cudss) (optional, when `PALACE_WITH_CUDSS=ON`)
   - [SLEPc](https://slepc.upv.es/) (optional, when `PALACE_WITH_SLEPC=ON`), including
     [PETSc](https://petsc.org/release/)
   - [ARPACK-NG](https://github.com/opencollab/arpack-ng) (optional, when
@@ -280,8 +284,8 @@ source code for these dependencies is downloaded during the build process:
   - [Eigen](https://eigen.tuxfamily.org)
 
 For solving eigenvalue problems, at least one of SLEPc or ARPACK-NG must be specified.
-Typically only one of the SuperLU_DIST, STRUMPACK, and MUMPS dependencies is required but
-all can be built so the user can decide at runtime which solver to use.
+Typically only one of the SuperLU_DIST, STRUMPACK, MUMPS, and cuDSS dependencies is
+required but all can be built so the user can decide at runtime which solver to use.
 
 For unit testing, *Palace* relies on the [Catch2
 library](https://github.com/catchorg/Catch2), which is automatically downloaded