Skip to content

Add GPU support to hypre solvers #14130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
victorapm wants to merge 2 commits into from
Closed

Add GPU support to hypre solvers #14130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8bc1723
Add GPU support to hypre solvers
victorapm Jan 28, 2025
87f6716
Remove mpi_f08 check
victorapm Jan 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 53 additions & 13 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
cmake_minimum_required(VERSION 3.24 FATAL_ERROR)

# Raise an error if the source and binary directories are the same
if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR})
message(FATAL_ERROR "In-place build not allowed! Please use a separate build directory.")
endif ()

project(
fds
VERSION 6.9.1
Expand All @@ -8,6 +13,9 @@ project(
enable_language(Fortran)

option(USE_HYPRE "Use the hypre library" ON)
option(USE_HYPRE_NVIDIA "Use the hypre library with NVIDIA GPUs" OFF)
option(USE_HYPRE_AMDGPU "Use the hypre library with AMD GPUs" OFF)
option(USE_HYPRE_INTELGPU "Use the hypre library with Intel GPUs" OFF)
option(USE_SYSTEM_HYPRE "Use the hypre library from the system" OFF)

option(USE_SUNDIALS "Use the sundials library" ON)
Expand Down Expand Up @@ -142,40 +150,61 @@ if(WIN32)
endif()

# Find or build hypre if the option is set to use it
if(USE_HYPRE)
if(USE_HYPRE OR USE_HYPRE_NVIDIA OR USE_HYPRE_AMDGPU OR USE_HYPRE_INTELGPU)
# Unless forced to use the system version, download and build hypre
if(NOT(USE_SYSTEM_HYPRE))
include(FetchContent)
# As we are not using the system hypre, we need to choose the version we
# want
set(HYPRE_GIT_VERSION "2.32.0" )
set(HYPRE_GIT_VERSION "v2.32.0-24-63331f19c" )
FetchContent_Declare(
HYPRE
GIT_REPOSITORY https://github.com/hypre-space/hypre.git
# Currently we need to refer directly to a commit which contains a
# patch to handle HYPRE_FMANGLE
GIT_TAG 6d01417697aa84d118eb1e1eb91315df42575be5 # v${HYPRE_GIT_VERSION}
GIT_TAG 63331f19c72b2ec363824907c0575a6edf7bbf3c # From: 01/27/25 # v${HYPRE_GIT_VERSION}
Copy link
Contributor Author

@victorapm victorapm Jan 28, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@marcosvanella @rmcdermo this is the minimum required version of hypre for this to work. This commit hash is located at hypre's master branch and will be added in the upcoming release

Copy link
Contributor

@rmcdermo rmcdermo Jan 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@victorapm OK, understood. Thanks! We can add to FDS master branch once there is a new hypre release.

SOURCE_SUBDIR src
OVERRIDE_FIND_PACKAGE
)

# Set some options for building hypre
if (WIN32)
set(HYPRE_ENABLE_FMANGLE CAPS)
endif()
if (USE_HYPRE_NVIDIA AND USE_HYPRE_AMDGPU)
message(FATAL_ERROR "Incompatible options: USE_HYPRE_NVIDIA and USE_HYPRE_AMDGPU")
elseif (USE_HYPRE_NVIDIA AND USE_HYPRE_INTELGPU)
message(FATAL_ERROR "Incompatible options: USE_HYPRE_NVIDIA and USE_HYPRE_INTELGPU")
elseif (USE_HYPRE_AMDGPU AND USE_HYPRE_INTELGPU)
message(FATAL_ERROR "Incompatible options: USE_HYPRE_AMDGPU and USE_HYPRE_INTELGPU")
endif()
if (USE_HYPRE_NVIDIA)
set(HYPRE_ENABLE_CUDA ON CACHE BOOL "" FORCE)
elseif (USE_HYPRE_AMDGPU)
set(HYPRE_ENABLE_HIP ON CACHE BOOL "" FORCE)
elseif (USE_HYPRE_INTELGPU)
set(HYPRE_ENABLE_SYCL ON CACHE BOOL "" FORCE)
endif()
if (USE_HYPRE_NVIDIA OR USE_HYPRE_AMDGPU OR USE_HYPRE_INTELGPU)
set(CMAKE_POSITION_INDEPEDENT_CODE ON CACHE BOOL "" FORCE)
endif()

message(STATUS "Fetching HYPRE")
FetchContent_MakeAvailable(HYPRE)
endif()

# Use hypre if found
find_package(HYPRE)
find_package(HYPRE 2.32.0 REQUIRED)
if(HYPRE_FOUND)
if(NOT(HYPRE_VERSION))
# We aren't using the system version so we'll revert to the git
# version
set(HYPRE_VERSION "${HYPRE_GIT_VERSION}")
endif()
target_link_libraries(fds PRIVATE HYPRE)
target_link_libraries(fds PRIVATE HYPRE::HYPRE)
target_compile_definitions(fds PRIVATE WITH_HYPRE)
target_compile_definitions(fds PRIVATE HYPRE_PP="${HYPRE_VERSION}")
if (USE_HYPRE_NVIDIA OR USE_HYPRE_AMDGPU OR USE_HYPRE_INTELGPU)
target_compile_definitions(fds PRIVATE WITH_HYPRE_DEVICE)
endif()
endif()
endif()

Expand All @@ -202,12 +231,23 @@ if(USE_SUNDIALS)
set(EXAMPLES_INSTALL OFF CACHE BOOL "" FORCE)
set(ENABLE_OPENMP ON CACHE BOOL "" FORCE)
set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)
# if(HYPRE_FOUND)
# set(ENABLE_HYPRE ON CACHE BOOL "" FORCE)
# FetchContent_MakeAvailable(HYPRE SUNDIALS)
# else()
FetchContent_MakeAvailable(SUNDIALS)
# endif()
if(HYPRE_FOUND)
if(NOT USE_SYSTEM_HYPRE)
set(ENABLE_HYPRE OFF CACHE BOOL "" FORCE)
else()
set(ENABLE_HYPRE ON CACHE BOOL "" FORCE)
endif()
if(HYPRE_ROOT)
set(HYPRE_DIR "${HYPRE_ROOT}")
endif()
if(HYPRE_DIR)
message(STATUS "Sundials using HYPRE_DIR=${HYPRE_DIR}")
endif()
else()
set(ENABLE_HYPRE OFF CACHE BOOL "" FORCE)
endif()
message(STATUS "Fetching Sundials")
FetchContent_MakeAvailable(SUNDIALS)
endif()

# Use sundials if found
Expand Down
2 changes: 1 addition & 1 deletion Source/cons.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,7 @@ MODULE GLOBAL_CONSTANTS
INTEGER, PARAMETER :: MKL_PARDISO_FLAG=1 !< Integer matrix solver library flag for MKL PARDISO
INTEGER, PARAMETER :: MKL_CPARDISO_FLAG=1 !< Integer matrix solver library flag for MKL CLUSTER PARDISO
INTEGER, PARAMETER :: HYPRE_FLAG=2 !< Integer matrix solver library flag for HYPRE
LOGICAL :: HYPRE_DEVICE_RUN=.TRUE. !< Offload hypre solver to GPU if available, otherwise run on CPU
INTEGER :: ULMAT_SOLVER_LIBRARY=MKL_PARDISO_FLAG !< Integer ULMAT library flag (defaults to MKL PARDISO)
INTEGER :: UGLMAT_SOLVER_LIBRARY=MKL_CPARDISO_FLAG !< Integer UGLMAT library flag (defaults to MKL CPARDISO)
INTEGER :: PRES_FLAG = FFT_FLAG !< Pressure solver
Expand Down Expand Up @@ -886,4 +887,3 @@ MODULE CHEMCONS


END MODULE CHEMCONS

13 changes: 8 additions & 5 deletions Source/func.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7859,9 +7859,9 @@ SUBROUTINE REALLOCATE_BOUNDARY_ONE_D(ONE_D)
IF (ALLOCATED(ONE_D%TMP)) DEALLOCATE(ONE_D%TMP) ; ALLOCATE(ONE_D%TMP(0:ONE_D%N_CELLS_MAX+1))
IF (ALLOCATED(ONE_D%DELTA_TMP)) DEALLOCATE(ONE_D%DELTA_TMP) ; ALLOCATE(ONE_D%DELTA_TMP(0:ONE_D%N_CELLS_MAX+1))
IF (ALLOCATED(ONE_D%LAYER_THICKNESS)) DEALLOCATE(ONE_D%LAYER_THICKNESS) ; ALLOCATE(ONE_D%LAYER_THICKNESS(ONE_D%N_LAYERS))
IF (ALLOCATED(ONE_D%LAYER_THICKNESS_OLD)) DEALLOCATE(ONE_D%LAYER_THICKNESS_OLD)
IF (ALLOCATED(ONE_D%LAYER_THICKNESS_OLD)) DEALLOCATE(ONE_D%LAYER_THICKNESS_OLD)
ALLOCATE(ONE_D%LAYER_THICKNESS_OLD(ONE_D%N_LAYERS))
IF (ALLOCATED(ONE_D%MIN_LAYER_THICKNESS)) DEALLOCATE(ONE_D%MIN_LAYER_THICKNESS)
IF (ALLOCATED(ONE_D%MIN_LAYER_THICKNESS)) DEALLOCATE(ONE_D%MIN_LAYER_THICKNESS)
ALLOCATE(ONE_D%MIN_LAYER_THICKNESS(ONE_D%N_LAYERS))
IF (ALLOCATED(ONE_D%HT3D_LAYER)) DEALLOCATE(ONE_D%HT3D_LAYER) ; ALLOCATE(ONE_D%HT3D_LAYER(ONE_D%N_LAYERS))
IF (ALLOCATED(ONE_D%MIN_DIFFUSIVITY)) DEALLOCATE(ONE_D%MIN_DIFFUSIVITY) ; ALLOCATE(ONE_D%MIN_DIFFUSIVITY(ONE_D%N_LAYERS))
Expand Down Expand Up @@ -9824,9 +9824,13 @@ SUBROUTINE WRITE_SUMMARY_INFO(LU,INPUT_FILE_INCLUDED)
WRITE(LU,'(/A,I1,A,I1)') ' MPI version: ',MPIVERSION,'.',MPISUBVERSION
WRITE(LU,'(A,A)') ' MPI library version: ',TRIM(MPILIBVERSION)
#ifdef HYPRE_PP
WRITE(LU,'(A,A)') ' Hypre library version: ',TRIM(HYPRE_PP)
#ifdef WITH_HYPRE_DEVICE
WRITE(LU,'(A,A,A)') ' Hypre library version: ', TRIM(HYPRE_PP), ' (GPU support ON)'
#else
WRITE(LU,'(A)') ' Hypre library: not used '
WRITE(LU,'(A,A,A)') ' Hypre library version: ', TRIM(HYPRE_PP), ' (GPU support OFF)'
#endif
#else
WRITE(LU,'(A)') ' Hypre library: not used'
#endif
#ifdef SUNDIALS_PP
WRITE(LU,'(A,A)') ' Sundials library version: ',TRIM(SUNDIALS_PP)
Expand Down Expand Up @@ -10028,4 +10032,3 @@ SUBROUTINE ACCUMULATE_STRING(STRING_SIZE,MYSTR,ACCSTR,ACCSTR_T_LEN,ACCSTR_USE_LE
END SUBROUTINE ACCUMULATE_STRING

END MODULE MISC_FUNCTIONS

51 changes: 43 additions & 8 deletions Source/imkl.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,26 +209,27 @@ MODULE HYPRE_INTERFACE
INTEGER, PARAMETER :: HYPRE_SOLVER_SETPRINTLEVEL = 0 ! 0=no output, 1=minimal, 2=verbose
INTEGER, PARAMETER :: HYPRE_SOLVER_SETLOGGING = 0 ! 0=no logging, 1=solver stores intermediate info, norms, etc.
INTEGER, PARAMETER :: HYPRE_PRECOND_SETPRINTLEVEL = 0 ! 0=no output, 1=minimal, 2=verbose
INTEGER, PARAMETER :: HYPRE_PRECOND_COARSENINGTYPE = 10 ! 0 CLJP (Cleary-Luby-Jones-Plassmann) parallel coarsening
INTEGER, PARAMETER :: HYPRE_PRECOND_COARSENINGTYPE = 8 ! 0 CLJP (Cleary-Luby-Jones-Plassmann) parallel coarsening
! 1 Classical Ruge-Stüben (RS) coarsening
! 3 Modified RS coarsening
! 6 Falgout coarsening (a combination of CLJP and RS)
! 8 PMIS (Parallel Modified Independent Set) coarsening
! 10 HMIS (Hybrid Modified Independent Set) coarsening
! 21 Falgout coarsening with aggressive coarsening
INTEGER, PARAMETER :: HYPRE_PRECOND_SETRELAXTYPE = 8 ! 0 Jacobi (default)
! 1 Gauss-Seidel, sequential (very slow in parallel)
INTEGER, PARAMETER :: HYPRE_PRECOND_SETRELAXTYPE = 18 ! 1 Gauss-Seidel, sequential (very slow in parallel)
! 2 Gauss-Seidel, interior points first (parallel variant)
! 3 Hybrid Gauss-Seidel or SOR (symmetric in parallel)
! 6 L1-scaled Jacobi
! 3 hybrid Gauss-Seidel or SOR, forward solve
! 4 hybrid Gauss-Seidel or SOR, backward solve
! 6 Hybrid symmetric Gauss-Seidel or SSOR
! 7 Jacobi
! 8 L1-scaled hybrid symmetric Gauss-Seidel/SOR
! 13 Two-step Jacobi
! 16 Chebyshev smoothing (useful for difficult problems)
! 18 L1-scaled Jacobi
INTEGER, PARAMETER :: HYPRE_PRECOND_NUMSWEEPS = 1 ! Number of sweeps on each level of preconditioner
REAL(KIND=8), PARAMETER :: HYPRE_PRECOND_TOL = 0.D0 ! Preconditioner convergence tolerance
INTEGER, PARAMETER :: HYPRE_PRECOND_MAXITER = 1 ! Max number of iterations for preconditioner


TYPE HYPRE_ZM_TYPE
INTEGER(KIND=8) :: A_H ! Matrix handle
INTEGER(KIND=8) :: F_H ! RHS handle
Expand All @@ -253,15 +254,20 @@ SUBROUTINE HYPRE_FINALIZE(IERR)
INTEGER, INTENT(OUT) :: IERR
END SUBROUTINE HYPRE_FINALIZE

SUBROUTINE HYPRE_IJMATRIXCREATE(COMM,I1,I2,I3,I4,A,IERR)
SUBROUTINE HYPRE_SETEXECUTIONPOLICY(EXEC_POLICY, IERR)
INTEGER, INTENT(IN) :: EXEC_POLICY
INTEGER, INTENT(OUT) :: IERR
END SUBROUTINE HYPRE_SETEXECUTIONPOLICY

SUBROUTINE HYPRE_IJMATRIXCREATE(COMM, I1, I2, I3, I4, A, IERR)
USE MPI_F08
TYPE(MPI_COMM), INTENT(IN) :: COMM
INTEGER, INTENT(IN) :: I1,I2,I3,I4
INTEGER(KIND=8), INTENT(OUT) :: A
INTEGER, INTENT(OUT) :: IERR
END SUBROUTINE HYPRE_IJMATRIXCREATE

SUBROUTINE HYPRE_IJMATRIXSETOBJECTTYPE(A,HYPRE_PARCSR_INT,IERR)
SUBROUTINE HYPRE_IJMATRIXSETOBJECTTYPE(A, HYPRE_PARCSR_INT, IERR)
INTEGER(KIND=8), INTENT(IN) :: A
INTEGER, INTENT(IN) :: HYPRE_PARCSR_INT
INTEGER, INTENT(OUT) :: IERR
Expand All @@ -272,6 +278,12 @@ SUBROUTINE HYPRE_IJMATRIXINITIALIZE(A,IERR)
INTEGER, INTENT(OUT) :: IERR
END SUBROUTINE HYPRE_IJMATRIXINITIALIZE

SUBROUTINE HYPRE_IJMATRIXINITIALIZE_V2(A, MEMORY_LOC, IERR)
INTEGER(KIND=8), INTENT(IN) :: A
INTEGER, INTENT(IN) :: MEMORY_LOC
INTEGER, INTENT(OUT) :: IERR
END SUBROUTINE HYPRE_IJMATRIXINITIALIZE_V2

SUBROUTINE HYPRE_IJMATRIXSETVALUES(A, NROWS, NCOLS, ROWS, COLS, VALUES, IERR)
INTEGER(KIND=8), INTENT(IN) :: A
INTEGER, INTENT(IN) :: NROWS
Expand All @@ -282,6 +294,12 @@ SUBROUTINE HYPRE_IJMATRIXSETVALUES(A, NROWS, NCOLS, ROWS, COLS, VALUES, IERR)
INTEGER, INTENT(OUT) :: IERR
END SUBROUTINE HYPRE_IJMATRIXSETVALUES

SUBROUTINE HYPRE_IJMATRIXMIGRATE(A, MEMORY_LOC, IERR)
INTEGER(KIND=8), INTENT(IN) :: A
INTEGER, INTENT(IN) :: MEMORY_LOC
INTEGER, INTENT(OUT) :: IERR
END SUBROUTINE HYPRE_IJMATRIXMIGRATE

SUBROUTINE HYPRE_IJMATRIXASSEMBLE(A,IERR)
INTEGER(KIND=8), INTENT(IN) :: A
INTEGER, INTENT(OUT) :: IERR
Expand Down Expand Up @@ -312,6 +330,12 @@ SUBROUTINE HYPRE_IJVECTORINITIALIZE(X, IERR)
INTEGER, INTENT(OUT) :: IERR
END SUBROUTINE HYPRE_IJVECTORINITIALIZE

SUBROUTINE HYPRE_IJVECTORINITIALIZE_V2(X, MEMORY_LOC, IERR)
INTEGER(KIND=8), INTENT(IN) :: X
INTEGER, INTENT(IN) :: MEMORY_LOC
INTEGER, INTENT(OUT) :: IERR
END SUBROUTINE HYPRE_IJVECTORINITIALIZE_V2

SUBROUTINE HYPRE_IJVECTORSETVALUES(X, LOCAL_SIZE, ROWS, VALUES, IERR)
INTEGER(KIND=8), INTENT(IN) :: X
INTEGER, INTENT(IN) :: LOCAL_SIZE
Expand All @@ -328,6 +352,12 @@ SUBROUTINE HYPRE_IJVECTORGETVALUES(X, LOCAL_SIZE, ROWS, VALUES, IERR)
INTEGER, INTENT(OUT) :: IERR
END SUBROUTINE HYPRE_IJVECTORGETVALUES

SUBROUTINE HYPRE_IJVECTORMIGRATE(A, MEMORY_LOC, IERR)
INTEGER(KIND=8), INTENT(IN) :: A
INTEGER, INTENT(IN) :: MEMORY_LOC
INTEGER, INTENT(OUT) :: IERR
END SUBROUTINE HYPRE_IJVECTORMIGRATE

SUBROUTINE HYPRE_IJVECTORASSEMBLE(X, IERR)
INTEGER(KIND=8), INTENT(IN) :: X
INTEGER, INTENT(OUT) :: IERR
Expand Down Expand Up @@ -498,17 +528,22 @@ END SUBROUTINE HYPRE_PARCSRPCGGETFINALRELATIVE
HYPRE_EXEC_DEVICE, & ! from HYPREf.h
HYPRE_INITIALIZE, & ! subroutine in HYPRE library
HYPRE_FINALIZE, & ! subroutine in HYPRE library
HYPRE_SETEXECUTIONPOLICY, & ! subroutine in HYPRE library
HYPRE_IJMATRIXCREATE, & ! subroutine in HYPRE library
HYPRE_IJMATRIXSETOBJECTTYPE, & ! subroutine in HYPRE library
HYPRE_IJMATRIXINITIALIZE, & ! subroutine in HYPRE library
HYPRE_IJMATRIXINITIALIZE_V2, & ! subroutine in HYPRE library
HYPRE_IJMATRIXSETVALUES, & ! subroutine in HYPRE library
HYPRE_IJMATRIXMIGRATE, & ! subroutine in HYPRE library
HYPRE_IJMATRIXASSEMBLE, & ! subroutine in HYPRE library
HYPRE_IJMATRIXGETOBJECT, & ! subroutine in HYPRE library
HYPRE_IJVECTORCREATE, & ! subroutine in HYPRE library
HYPRE_IJVECTORSETOBJECTTYPE, & ! subroutine in HYPRE library
HYPRE_IJVECTORINITIALIZE, & ! subroutine in HYPRE library
HYPRE_IJVECTORINITIALIZE_V2, & ! subroutine in HYPRE library
HYPRE_IJVECTORSETVALUES, & ! subroutine in HYPRE library
HYPRE_IJVECTORGETVALUES, & ! subroutine in HYPRE library
HYPRE_IJVECTORMIGRATE, & ! subroutine in HYPRE library
HYPRE_IJVECTORASSEMBLE, & ! subroutine in HYPRE library
HYPRE_IJVECTORGETOBJECT, & ! subroutine in HYPRE library
HYPRE_PARCSRPCGCREATE, & ! subroutine in HYPRE library
Expand Down
Loading
Loading