diff --git a/CMakeLists.txt b/CMakeLists.txt index 06173b92741..157b7c9888c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,10 @@ cmake_minimum_required(VERSION 3.24 FATAL_ERROR) +# Raise an error if the source and binary directories are the same +if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) + message(FATAL_ERROR "In-place build not allowed! Please use a separate build directory.") +endif () + project( fds VERSION 6.9.1 @@ -8,6 +13,9 @@ project( enable_language(Fortran) option(USE_HYPRE "Use the hypre library" ON) +option(USE_HYPRE_NVIDIA "Use the hypre library with NVIDIA GPUs" OFF) +option(USE_HYPRE_AMDGPU "Use the hypre library with AMD GPUs" OFF) +option(USE_HYPRE_INTELGPU "Use the hypre library with Intel GPUs" OFF) option(USE_SYSTEM_HYPRE "Use the hypre library from the system" OFF) option(USE_SUNDIALS "Use the sundials library" ON) @@ -142,40 +150,61 @@ if(WIN32) endif() # Find or build hypre if the option is set to use it -if(USE_HYPRE) +if(USE_HYPRE OR USE_HYPRE_NVIDIA OR USE_HYPRE_AMDGPU OR USE_HYPRE_INTELGPU) # Unless forced to use the system version, download and build hypre if(NOT(USE_SYSTEM_HYPRE)) include(FetchContent) # As we are not using the system hypre, we need to choose the version we # want - set(HYPRE_GIT_VERSION "2.32.0" ) + set(HYPRE_GIT_VERSION "v2.32.0-24-63331f19c" ) FetchContent_Declare( HYPRE GIT_REPOSITORY https://github.com/hypre-space/hypre.git - # Currently we need to refer directly to a commit which contains a - # patch to handle HYPRE_FMANGLE - GIT_TAG 6d01417697aa84d118eb1e1eb91315df42575be5 # v${HYPRE_GIT_VERSION} + GIT_TAG 63331f19c72b2ec363824907c0575a6edf7bbf3c # From: 01/27/25 # v${HYPRE_GIT_VERSION} SOURCE_SUBDIR src OVERRIDE_FIND_PACKAGE ) + # Set some options for building hypre if (WIN32) set(HYPRE_ENABLE_FMANGLE CAPS) endif() + if (USE_HYPRE_NVIDIA AND USE_HYPRE_AMDGPU) + message(FATAL_ERROR "Incompatible options: USE_HYPRE_NVIDIA and USE_HYPRE_AMDGPU") + elseif (USE_HYPRE_NVIDIA AND USE_HYPRE_INTELGPU) + message(FATAL_ERROR "Incompatible options: USE_HYPRE_NVIDIA and USE_HYPRE_INTELGPU") + elseif (USE_HYPRE_AMDGPU AND USE_HYPRE_INTELGPU) + message(FATAL_ERROR "Incompatible options: USE_HYPRE_AMDGPU and USE_HYPRE_INTELGPU") + endif() + if (USE_HYPRE_NVIDIA) + set(HYPRE_ENABLE_CUDA ON CACHE BOOL "" FORCE) + elseif (USE_HYPRE_AMDGPU) + set(HYPRE_ENABLE_HIP ON CACHE BOOL "" FORCE) + elseif (USE_HYPRE_INTELGPU) + set(HYPRE_ENABLE_SYCL ON CACHE BOOL "" FORCE) + endif() + if (USE_HYPRE_NVIDIA OR USE_HYPRE_AMDGPU OR USE_HYPRE_INTELGPU) + set(CMAKE_POSITION_INDEPEDENT_CODE ON CACHE BOOL "" FORCE) + endif() + + message(STATUS "Fetching HYPRE") FetchContent_MakeAvailable(HYPRE) endif() # Use hypre if found - find_package(HYPRE) + find_package(HYPRE 2.32.0 REQUIRED) if(HYPRE_FOUND) if(NOT(HYPRE_VERSION)) # We aren't using the system version so we'll revert to the git # version set(HYPRE_VERSION "${HYPRE_GIT_VERSION}") endif() - target_link_libraries(fds PRIVATE HYPRE) + target_link_libraries(fds PRIVATE HYPRE::HYPRE) target_compile_definitions(fds PRIVATE WITH_HYPRE) target_compile_definitions(fds PRIVATE HYPRE_PP="${HYPRE_VERSION}") + if (USE_HYPRE_NVIDIA OR USE_HYPRE_AMDGPU OR USE_HYPRE_INTELGPU) + target_compile_definitions(fds PRIVATE WITH_HYPRE_DEVICE) + endif() endif() endif() @@ -202,12 +231,23 @@ if(USE_SUNDIALS) set(EXAMPLES_INSTALL OFF CACHE BOOL "" FORCE) set(ENABLE_OPENMP ON CACHE BOOL "" FORCE) set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE) - # if(HYPRE_FOUND) - # set(ENABLE_HYPRE ON CACHE BOOL "" FORCE) - # FetchContent_MakeAvailable(HYPRE SUNDIALS) - # else() - FetchContent_MakeAvailable(SUNDIALS) - # endif() + if(HYPRE_FOUND) + if(NOT USE_SYSTEM_HYPRE) + set(ENABLE_HYPRE OFF CACHE BOOL "" FORCE) + else() + set(ENABLE_HYPRE ON CACHE BOOL "" FORCE) + endif() + if(HYPRE_ROOT) + set(HYPRE_DIR "${HYPRE_ROOT}") + endif() + if(HYPRE_DIR) + message(STATUS "Sundials using HYPRE_DIR=${HYPRE_DIR}") + endif() + else() + set(ENABLE_HYPRE OFF CACHE BOOL "" FORCE) + endif() + message(STATUS "Fetching Sundials") + FetchContent_MakeAvailable(SUNDIALS) endif() # Use sundials if found diff --git a/Source/cons.f90 b/Source/cons.f90 index 5e058ec7765..e028cb91bfb 100644 --- a/Source/cons.f90 +++ b/Source/cons.f90 @@ -523,6 +523,7 @@ MODULE GLOBAL_CONSTANTS INTEGER, PARAMETER :: MKL_PARDISO_FLAG=1 !< Integer matrix solver library flag for MKL PARDISO INTEGER, PARAMETER :: MKL_CPARDISO_FLAG=1 !< Integer matrix solver library flag for MKL CLUSTER PARDISO INTEGER, PARAMETER :: HYPRE_FLAG=2 !< Integer matrix solver library flag for HYPRE +LOGICAL :: HYPRE_DEVICE_RUN=.TRUE. !< Offload hypre solver to GPU if available, otherwise run on CPU INTEGER :: ULMAT_SOLVER_LIBRARY=MKL_PARDISO_FLAG !< Integer ULMAT library flag (defaults to MKL PARDISO) INTEGER :: UGLMAT_SOLVER_LIBRARY=MKL_CPARDISO_FLAG !< Integer UGLMAT library flag (defaults to MKL CPARDISO) INTEGER :: PRES_FLAG = FFT_FLAG !< Pressure solver @@ -886,4 +887,3 @@ MODULE CHEMCONS END MODULE CHEMCONS - diff --git a/Source/func.f90 b/Source/func.f90 index 0cc29f62ea8..6c1cfc46cfa 100644 --- a/Source/func.f90 +++ b/Source/func.f90 @@ -7859,9 +7859,9 @@ SUBROUTINE REALLOCATE_BOUNDARY_ONE_D(ONE_D) IF (ALLOCATED(ONE_D%TMP)) DEALLOCATE(ONE_D%TMP) ; ALLOCATE(ONE_D%TMP(0:ONE_D%N_CELLS_MAX+1)) IF (ALLOCATED(ONE_D%DELTA_TMP)) DEALLOCATE(ONE_D%DELTA_TMP) ; ALLOCATE(ONE_D%DELTA_TMP(0:ONE_D%N_CELLS_MAX+1)) IF (ALLOCATED(ONE_D%LAYER_THICKNESS)) DEALLOCATE(ONE_D%LAYER_THICKNESS) ; ALLOCATE(ONE_D%LAYER_THICKNESS(ONE_D%N_LAYERS)) -IF (ALLOCATED(ONE_D%LAYER_THICKNESS_OLD)) DEALLOCATE(ONE_D%LAYER_THICKNESS_OLD) +IF (ALLOCATED(ONE_D%LAYER_THICKNESS_OLD)) DEALLOCATE(ONE_D%LAYER_THICKNESS_OLD) ALLOCATE(ONE_D%LAYER_THICKNESS_OLD(ONE_D%N_LAYERS)) -IF (ALLOCATED(ONE_D%MIN_LAYER_THICKNESS)) DEALLOCATE(ONE_D%MIN_LAYER_THICKNESS) +IF (ALLOCATED(ONE_D%MIN_LAYER_THICKNESS)) DEALLOCATE(ONE_D%MIN_LAYER_THICKNESS) ALLOCATE(ONE_D%MIN_LAYER_THICKNESS(ONE_D%N_LAYERS)) IF (ALLOCATED(ONE_D%HT3D_LAYER)) DEALLOCATE(ONE_D%HT3D_LAYER) ; ALLOCATE(ONE_D%HT3D_LAYER(ONE_D%N_LAYERS)) IF (ALLOCATED(ONE_D%MIN_DIFFUSIVITY)) DEALLOCATE(ONE_D%MIN_DIFFUSIVITY) ; ALLOCATE(ONE_D%MIN_DIFFUSIVITY(ONE_D%N_LAYERS)) @@ -9824,9 +9824,13 @@ SUBROUTINE WRITE_SUMMARY_INFO(LU,INPUT_FILE_INCLUDED) WRITE(LU,'(/A,I1,A,I1)') ' MPI version: ',MPIVERSION,'.',MPISUBVERSION WRITE(LU,'(A,A)') ' MPI library version: ',TRIM(MPILIBVERSION) #ifdef HYPRE_PP -WRITE(LU,'(A,A)') ' Hypre library version: ',TRIM(HYPRE_PP) +#ifdef WITH_HYPRE_DEVICE +WRITE(LU,'(A,A,A)') ' Hypre library version: ', TRIM(HYPRE_PP), ' (GPU support ON)' #else -WRITE(LU,'(A)') ' Hypre library: not used ' +WRITE(LU,'(A,A,A)') ' Hypre library version: ', TRIM(HYPRE_PP), ' (GPU support OFF)' +#endif +#else +WRITE(LU,'(A)') ' Hypre library: not used' #endif #ifdef SUNDIALS_PP WRITE(LU,'(A,A)') ' Sundials library version: ',TRIM(SUNDIALS_PP) @@ -10028,4 +10032,3 @@ SUBROUTINE ACCUMULATE_STRING(STRING_SIZE,MYSTR,ACCSTR,ACCSTR_T_LEN,ACCSTR_USE_LE END SUBROUTINE ACCUMULATE_STRING END MODULE MISC_FUNCTIONS - diff --git a/Source/imkl.f90 b/Source/imkl.f90 index edcd34fbff2..e0f045c3b06 100644 --- a/Source/imkl.f90 +++ b/Source/imkl.f90 @@ -209,26 +209,27 @@ MODULE HYPRE_INTERFACE INTEGER, PARAMETER :: HYPRE_SOLVER_SETPRINTLEVEL = 0 ! 0=no output, 1=minimal, 2=verbose INTEGER, PARAMETER :: HYPRE_SOLVER_SETLOGGING = 0 ! 0=no logging, 1=solver stores intermediate info, norms, etc. INTEGER, PARAMETER :: HYPRE_PRECOND_SETPRINTLEVEL = 0 ! 0=no output, 1=minimal, 2=verbose - INTEGER, PARAMETER :: HYPRE_PRECOND_COARSENINGTYPE = 10 ! 0 CLJP (Cleary-Luby-Jones-Plassmann) parallel coarsening + INTEGER, PARAMETER :: HYPRE_PRECOND_COARSENINGTYPE = 8 ! 0 CLJP (Cleary-Luby-Jones-Plassmann) parallel coarsening ! 1 Classical Ruge-Stüben (RS) coarsening ! 3 Modified RS coarsening ! 6 Falgout coarsening (a combination of CLJP and RS) ! 8 PMIS (Parallel Modified Independent Set) coarsening ! 10 HMIS (Hybrid Modified Independent Set) coarsening ! 21 Falgout coarsening with aggressive coarsening - INTEGER, PARAMETER :: HYPRE_PRECOND_SETRELAXTYPE = 8 ! 0 Jacobi (default) - ! 1 Gauss-Seidel, sequential (very slow in parallel) + INTEGER, PARAMETER :: HYPRE_PRECOND_SETRELAXTYPE = 18 ! 1 Gauss-Seidel, sequential (very slow in parallel) ! 2 Gauss-Seidel, interior points first (parallel variant) - ! 3 Hybrid Gauss-Seidel or SOR (symmetric in parallel) - ! 6 L1-scaled Jacobi + ! 3 hybrid Gauss-Seidel or SOR, forward solve + ! 4 hybrid Gauss-Seidel or SOR, backward solve + ! 6 Hybrid symmetric Gauss-Seidel or SSOR + ! 7 Jacobi ! 8 L1-scaled hybrid symmetric Gauss-Seidel/SOR ! 13 Two-step Jacobi ! 16 Chebyshev smoothing (useful for difficult problems) + ! 18 L1-scaled Jacobi INTEGER, PARAMETER :: HYPRE_PRECOND_NUMSWEEPS = 1 ! Number of sweeps on each level of preconditioner REAL(KIND=8), PARAMETER :: HYPRE_PRECOND_TOL = 0.D0 ! Preconditioner convergence tolerance INTEGER, PARAMETER :: HYPRE_PRECOND_MAXITER = 1 ! Max number of iterations for preconditioner - TYPE HYPRE_ZM_TYPE INTEGER(KIND=8) :: A_H ! Matrix handle INTEGER(KIND=8) :: F_H ! RHS handle @@ -253,7 +254,12 @@ SUBROUTINE HYPRE_FINALIZE(IERR) INTEGER, INTENT(OUT) :: IERR END SUBROUTINE HYPRE_FINALIZE - SUBROUTINE HYPRE_IJMATRIXCREATE(COMM,I1,I2,I3,I4,A,IERR) + SUBROUTINE HYPRE_SETEXECUTIONPOLICY(EXEC_POLICY, IERR) + INTEGER, INTENT(IN) :: EXEC_POLICY + INTEGER, INTENT(OUT) :: IERR + END SUBROUTINE HYPRE_SETEXECUTIONPOLICY + + SUBROUTINE HYPRE_IJMATRIXCREATE(COMM, I1, I2, I3, I4, A, IERR) USE MPI_F08 TYPE(MPI_COMM), INTENT(IN) :: COMM INTEGER, INTENT(IN) :: I1,I2,I3,I4 @@ -261,7 +267,7 @@ SUBROUTINE HYPRE_IJMATRIXCREATE(COMM,I1,I2,I3,I4,A,IERR) INTEGER, INTENT(OUT) :: IERR END SUBROUTINE HYPRE_IJMATRIXCREATE - SUBROUTINE HYPRE_IJMATRIXSETOBJECTTYPE(A,HYPRE_PARCSR_INT,IERR) + SUBROUTINE HYPRE_IJMATRIXSETOBJECTTYPE(A, HYPRE_PARCSR_INT, IERR) INTEGER(KIND=8), INTENT(IN) :: A INTEGER, INTENT(IN) :: HYPRE_PARCSR_INT INTEGER, INTENT(OUT) :: IERR @@ -272,6 +278,12 @@ SUBROUTINE HYPRE_IJMATRIXINITIALIZE(A,IERR) INTEGER, INTENT(OUT) :: IERR END SUBROUTINE HYPRE_IJMATRIXINITIALIZE + SUBROUTINE HYPRE_IJMATRIXINITIALIZE_V2(A, MEMORY_LOC, IERR) + INTEGER(KIND=8), INTENT(IN) :: A + INTEGER, INTENT(IN) :: MEMORY_LOC + INTEGER, INTENT(OUT) :: IERR + END SUBROUTINE HYPRE_IJMATRIXINITIALIZE_V2 + SUBROUTINE HYPRE_IJMATRIXSETVALUES(A, NROWS, NCOLS, ROWS, COLS, VALUES, IERR) INTEGER(KIND=8), INTENT(IN) :: A INTEGER, INTENT(IN) :: NROWS @@ -282,6 +294,12 @@ SUBROUTINE HYPRE_IJMATRIXSETVALUES(A, NROWS, NCOLS, ROWS, COLS, VALUES, IERR) INTEGER, INTENT(OUT) :: IERR END SUBROUTINE HYPRE_IJMATRIXSETVALUES + SUBROUTINE HYPRE_IJMATRIXMIGRATE(A, MEMORY_LOC, IERR) + INTEGER(KIND=8), INTENT(IN) :: A + INTEGER, INTENT(IN) :: MEMORY_LOC + INTEGER, INTENT(OUT) :: IERR + END SUBROUTINE HYPRE_IJMATRIXMIGRATE + SUBROUTINE HYPRE_IJMATRIXASSEMBLE(A,IERR) INTEGER(KIND=8), INTENT(IN) :: A INTEGER, INTENT(OUT) :: IERR @@ -312,6 +330,12 @@ SUBROUTINE HYPRE_IJVECTORINITIALIZE(X, IERR) INTEGER, INTENT(OUT) :: IERR END SUBROUTINE HYPRE_IJVECTORINITIALIZE + SUBROUTINE HYPRE_IJVECTORINITIALIZE_V2(X, MEMORY_LOC, IERR) + INTEGER(KIND=8), INTENT(IN) :: X + INTEGER, INTENT(IN) :: MEMORY_LOC + INTEGER, INTENT(OUT) :: IERR + END SUBROUTINE HYPRE_IJVECTORINITIALIZE_V2 + SUBROUTINE HYPRE_IJVECTORSETVALUES(X, LOCAL_SIZE, ROWS, VALUES, IERR) INTEGER(KIND=8), INTENT(IN) :: X INTEGER, INTENT(IN) :: LOCAL_SIZE @@ -328,6 +352,12 @@ SUBROUTINE HYPRE_IJVECTORGETVALUES(X, LOCAL_SIZE, ROWS, VALUES, IERR) INTEGER, INTENT(OUT) :: IERR END SUBROUTINE HYPRE_IJVECTORGETVALUES + SUBROUTINE HYPRE_IJVECTORMIGRATE(A, MEMORY_LOC, IERR) + INTEGER(KIND=8), INTENT(IN) :: A + INTEGER, INTENT(IN) :: MEMORY_LOC + INTEGER, INTENT(OUT) :: IERR + END SUBROUTINE HYPRE_IJVECTORMIGRATE + SUBROUTINE HYPRE_IJVECTORASSEMBLE(X, IERR) INTEGER(KIND=8), INTENT(IN) :: X INTEGER, INTENT(OUT) :: IERR @@ -498,17 +528,22 @@ END SUBROUTINE HYPRE_PARCSRPCGGETFINALRELATIVE HYPRE_EXEC_DEVICE, & ! from HYPREf.h HYPRE_INITIALIZE, & ! subroutine in HYPRE library HYPRE_FINALIZE, & ! subroutine in HYPRE library + HYPRE_SETEXECUTIONPOLICY, & ! subroutine in HYPRE library HYPRE_IJMATRIXCREATE, & ! subroutine in HYPRE library HYPRE_IJMATRIXSETOBJECTTYPE, & ! subroutine in HYPRE library HYPRE_IJMATRIXINITIALIZE, & ! subroutine in HYPRE library + HYPRE_IJMATRIXINITIALIZE_V2, & ! subroutine in HYPRE library HYPRE_IJMATRIXSETVALUES, & ! subroutine in HYPRE library + HYPRE_IJMATRIXMIGRATE, & ! subroutine in HYPRE library HYPRE_IJMATRIXASSEMBLE, & ! subroutine in HYPRE library HYPRE_IJMATRIXGETOBJECT, & ! subroutine in HYPRE library HYPRE_IJVECTORCREATE, & ! subroutine in HYPRE library HYPRE_IJVECTORSETOBJECTTYPE, & ! subroutine in HYPRE library HYPRE_IJVECTORINITIALIZE, & ! subroutine in HYPRE library + HYPRE_IJVECTORINITIALIZE_V2, & ! subroutine in HYPRE library HYPRE_IJVECTORSETVALUES, & ! subroutine in HYPRE library HYPRE_IJVECTORGETVALUES, & ! subroutine in HYPRE library + HYPRE_IJVECTORMIGRATE, & ! subroutine in HYPRE library HYPRE_IJVECTORASSEMBLE, & ! subroutine in HYPRE library HYPRE_IJVECTORGETOBJECT, & ! subroutine in HYPRE library HYPRE_PARCSRPCGCREATE, & ! subroutine in HYPRE library diff --git a/Source/pres.f90 b/Source/pres.f90 index cf9e65e5ceb..396cb4ab6b5 100644 --- a/Source/pres.f90 +++ b/Source/pres.f90 @@ -1155,10 +1155,21 @@ SUBROUTINE ULMAT_SOLVER_SETUP(NM) CALL HYPRE_INITIALIZE(IERR) IF (IERR==1) THEN - WRITE(LU_ERR,'(A)') 'Error: HYPRE pressure solver initialization error.' + IF(MY_RANK==0) WRITE(LU_ERR,'(A)') 'Error: HYPRE pressure solver initialization error.' STOP_STATUS = SETUP_STOP RETURN ENDIF + +! Set execution policy +#ifdef WITH_HYPRE_DEVICE +IF (HYPRE_DEVICE_RUN) THEN + CALL HYPRE_SETEXECUTIONPOLICY(HYPRE_EXEC_DEVICE, IERR) + IF (MY_RANK==0) WRITE(LU_ERR,*) 'HYPRE : Setting execution policy to device (GPU)' +ELSE + CALL HYPRE_SETEXECUTIONPOLICY(HYPRE_EXEC_HOST, IERR) + IF (MY_RANK==0) WRITE(LU_ERR,*) 'HYPRE : Setting execution policy to host (CPU)' +ENDIF +#endif #endif ! Factor to drop DY(J) in cylindrical coordinates. Soln assumes DTheta=1. @@ -1686,11 +1697,29 @@ SUBROUTINE ULMAT_SOLVE_ZONE(NM,IPZ) #ifdef WITH_HYPRE CALL HYPRE_IJVECTORSETVALUES(ZM%HYPRE_ZM%F_H, ZM%NUNKH, ZM%HYPRE_ZM%INDICES, ZM%F_H, HYPRE_IERR) CALL HYPRE_IJVECTORASSEMBLE(ZM%HYPRE_ZM%F_H, HYPRE_IERR) + + ! Move vectors to GPU RAM +#ifdef WITH_HYPRE_DEVICE + IF (HYPRE_DEVICE_RUN) THEN + CALL HYPRE_IJVECTORMIGRATE(ZM%HYPRE_ZM%F_H, HYPRE_MEMORY_DEVICE, HYPRE_IERR) + CALL HYPRE_IJVECTORMIGRATE(ZM%HYPRE_ZM%X_H, HYPRE_MEMORY_DEVICE, HYPRE_IERR) + ENDIF +#endif + CALL HYPRE_PARCSRPCGSOLVE(ZM%HYPRE_ZM%SOLVER, ZM%HYPRE_ZM%PARCSR_A_H, ZM%HYPRE_ZM%PAR_F_H, ZM%HYPRE_ZM%PAR_X_H, HYPRE_IERR) IF (CHECK_POISSON .AND. HYPRE_SOLVER_SETPRINTLEVEL>0) THEN CALL HYPRE_PARCSRPCGGETNUMITERATIONS(ZM%HYPRE_ZM%SOLVER, ZM%HYPRE_ZM%NUM_ITERATIONS, HYPRE_IERR) CALL HYPRE_PARCSRPCGGETFINALRELATIVE(ZM%HYPRE_ZM%SOLVER, ZM%HYPRE_ZM%FINAL_RES_NORM, HYPRE_IERR) ENDIF + + ! Move vectors to CPU RAM +#ifdef WITH_HYPRE_DEVICE + IF (HYPRE_DEVICE_RUN) THEN + CALL HYPRE_IJVECTORMIGRATE(ZM%HYPRE_ZM%F_H, HYPRE_MEMORY_HOST, HYPRE_IERR) + CALL HYPRE_IJVECTORMIGRATE(ZM%HYPRE_ZM%X_H, HYPRE_MEMORY_HOST, HYPRE_IERR) + ENDIF +#endif + CALL HYPRE_IJVECTORGETVALUES(ZM%HYPRE_ZM%X_H, ZM%NUNKH, ZM%HYPRE_ZM%INDICES, ZM%X_H, HYPRE_IERR) #endif END SELECT LIBRARY_SELECT @@ -2834,9 +2863,9 @@ SUBROUTINE ULMAT_H_MATRIX_SOLVER_SETUP(NM,IPZ) ! Follows example in https://github.com/hypre-space/hypre/blob/master/src/examples/ex5f.f ! Create matrix - CALL HYPRE_IJMATRIXCREATE(MPI_COMM_SELF,0,ZM%NUNKH-1,0,ZM%NUNKH-1,ZM%HYPRE_ZM%A_H,HYPRE_IERR) - CALL HYPRE_IJMATRIXSETOBJECTTYPE(ZM%HYPRE_ZM%A_H,HYPRE_PARCSR,HYPRE_IERR) - CALL HYPRE_IJMATRIXINITIALIZE(ZM%HYPRE_ZM%A_H,HYPRE_IERR) + CALL HYPRE_IJMATRIXCREATE(MPI_COMM_SELF, 0, ZM%NUNKH-1, 0, ZM%NUNKH-1, ZM%HYPRE_ZM%A_H, HYPRE_IERR) + CALL HYPRE_IJMATRIXSETOBJECTTYPE(ZM%HYPRE_ZM%A_H, HYPRE_PARCSR, HYPRE_IERR) + CALL HYPRE_IJMATRIXINITIALIZE_V2(ZM%HYPRE_ZM%A_H, HYPRE_MEMORY_HOST, HYPRE_IERR) JD_H_MAT = JD_H_MAT - 1 DO IROW=1,ZM%NUNKH ZM%HYPRE_ZM%INDICES(IROW)=IROW-1 @@ -2848,11 +2877,11 @@ SUBROUTINE ULMAT_H_MATRIX_SOLVER_SETUP(NM,IPZ) ! Create right hand side vector CALL HYPRE_IJVECTORCREATE(MPI_COMM_SELF, 0, ZM%NUNKH-1, ZM%HYPRE_ZM%F_H, HYPRE_IERR) CALL HYPRE_IJVECTORSETOBJECTTYPE(ZM%HYPRE_ZM%F_H, HYPRE_PARCSR, HYPRE_IERR) - CALL HYPRE_IJVECTORINITIALIZE(ZM%HYPRE_ZM%F_H, HYPRE_IERR) + CALL HYPRE_IJVECTORINITIALIZE_V2(ZM%HYPRE_ZM%F_H, HYPRE_MEMORY_HOST, HYPRE_IERR) ! Create solution vector CALL HYPRE_IJVECTORCREATE(MPI_COMM_SELF, 0, ZM%NUNKH-1, ZM%HYPRE_ZM%X_H, HYPRE_IERR) CALL HYPRE_IJVECTORSETOBJECTTYPE(ZM%HYPRE_ZM%X_H, HYPRE_PARCSR, HYPRE_IERR) - CALL HYPRE_IJVECTORINITIALIZE(ZM%HYPRE_ZM%X_H, HYPRE_IERR) + CALL HYPRE_IJVECTORINITIALIZE_V2(ZM%HYPRE_ZM%X_H, HYPRE_MEMORY_HOST, HYPRE_IERR) ! Set values CALL HYPRE_IJVECTORSETVALUES(ZM%HYPRE_ZM%F_H, ZM%NUNKH, ZM%HYPRE_ZM%INDICES, ZM%F_H, HYPRE_IERR) CALL HYPRE_IJVECTORSETVALUES(ZM%HYPRE_ZM%X_H, ZM%NUNKH, ZM%HYPRE_ZM%INDICES, ZM%X_H, HYPRE_IERR) @@ -2863,6 +2892,15 @@ SUBROUTINE ULMAT_H_MATRIX_SOLVER_SETUP(NM,IPZ) CALL HYPRE_IJVECTORGETOBJECT(ZM%HYPRE_ZM%F_H, ZM%HYPRE_ZM%PAR_F_H, HYPRE_IERR) CALL HYPRE_IJVECTORGETOBJECT(ZM%HYPRE_ZM%X_H, ZM%HYPRE_ZM%PAR_X_H, HYPRE_IERR) + ! Move data to GPU RAM +#ifdef WITH_HYPRE_DEVICE + IF (HYPRE_DEVICE_RUN) THEN + CALL HYPRE_IJMATRIXMIGRATE(ZM%HYPRE_ZM%A_H, HYPRE_MEMORY_DEVICE, HYPRE_IERR) + CALL HYPRE_IJVECTORMIGRATE(ZM%HYPRE_ZM%F_H, HYPRE_MEMORY_DEVICE, HYPRE_IERR) + CALL HYPRE_IJVECTORMIGRATE(ZM%HYPRE_ZM%X_H, HYPRE_MEMORY_DEVICE, HYPRE_IERR) + ENDIF +#endif + ! Create solver (Parallel Compressed Sparse Row Preconditioned Conjugate Gradient) CALL HYPRE_PARCSRPCGCREATE(MPI_COMM_SELF, ZM%HYPRE_ZM%SOLVER, HYPRE_IERR) CALL HYPRE_PARCSRPCGSETMAXITER(ZM%HYPRE_ZM%SOLVER, HYPRE_SOLVER_MAXIT, HYPRE_IERR) @@ -2880,9 +2918,18 @@ SUBROUTINE ULMAT_H_MATRIX_SOLVER_SETUP(NM,IPZ) CALL HYPRE_BOOMERAMGSETTOL(ZM%HYPRE_ZM%PRECOND, HYPRE_PRECOND_TOL, HYPRE_IERR) CALL HYPRE_BOOMERAMGSETMAXITER(ZM%HYPRE_ZM%PRECOND, HYPRE_PRECOND_MAXITER, HYPRE_IERR) CALL HYPRE_PARCSRPCGSETPRECOND(ZM%HYPRE_ZM%SOLVER, HYPRE_PRECOND_ID, ZM%HYPRE_ZM%PRECOND, HYPRE_IERR) + ! Solver setup CALL HYPRE_PARCSRPCGSETUP(ZM%HYPRE_ZM%SOLVER, ZM%HYPRE_ZM%PARCSR_A_H, ZM%HYPRE_ZM%PAR_F_H, ZM%HYPRE_ZM%PAR_X_H, HYPRE_IERR) + ! Move vectors to CPU RAM +#ifdef WITH_HYPRE_DEVICE + IF (HYPRE_DEVICE_RUN) THEN + CALL HYPRE_IJVECTORMIGRATE(ZM%HYPRE_ZM%F_H, HYPRE_MEMORY_HOST, HYPRE_IERR) + CALL HYPRE_IJVECTORMIGRATE(ZM%HYPRE_ZM%X_H, HYPRE_MEMORY_HOST, HYPRE_IERR) + ENDIF +#endif + ! Deallocate NNZ_H_MAT, D_H_MAT, JD_H_MAT: IF (ALLOCATED(NNZ_H_MAT)) DEALLOCATE(NNZ_H_MAT) IF (ALLOCATED(D_H_MAT)) DEALLOCATE(D_H_MAT) @@ -3160,12 +3207,30 @@ SUBROUTINE GLMAT_SOLVER(T,DT) ! Solve the system using HYPRE: CALL HYPRE_IJVECTORSETVALUES(ZSL%HYPRE_ZSL%F_H, ZSL%NUNKH_LOCAL, ZSL%HYPRE_ZSL%INDICES, ZSL%F_H, HYPRE_IERR) CALL HYPRE_IJVECTORASSEMBLE(ZSL%HYPRE_ZSL%F_H, HYPRE_IERR) + + ! Move vectors to GPU RAM +#ifdef WITH_HYPRE_DEVICE + IF (HYPRE_DEVICE_RUN) THEN + CALL HYPRE_IJVECTORMIGRATE(ZSL%HYPRE_ZSL%F_H, HYPRE_MEMORY_DEVICE, HYPRE_IERR) + CALL HYPRE_IJVECTORMIGRATE(ZSL%HYPRE_ZSL%X_H, HYPRE_MEMORY_DEVICE, HYPRE_IERR) + ENDIF +#endif + CALL HYPRE_PARCSRPCGSOLVE(ZSL%HYPRE_ZSL%SOLVER, ZSL%HYPRE_ZSL%PARCSR_A_H, ZSL%HYPRE_ZSL%PAR_F_H, & ZSL%HYPRE_ZSL%PAR_X_H, HYPRE_IERR) IF (CHECK_POISSON .AND. HYPRE_SOLVER_SETPRINTLEVEL>0) THEN CALL HYPRE_PARCSRPCGGETNUMITERATIONS(ZSL%HYPRE_ZSL%SOLVER, ZSL%HYPRE_ZSL%NUM_ITERATIONS, HYPRE_IERR) CALL HYPRE_PARCSRPCGGETFINALRELATIVE(ZSL%HYPRE_ZSL%SOLVER, ZSL%HYPRE_ZSL%FINAL_RES_NORM, HYPRE_IERR) ENDIF + + ! Move vectors to CPU RAM +#ifdef WITH_HYPRE_DEVICE + IF (HYPRE_DEVICE_RUN) THEN + CALL HYPRE_IJVECTORMIGRATE(ZSL%HYPRE_ZSL%F_H, HYPRE_MEMORY_HOST, HYPRE_IERR) + CALL HYPRE_IJVECTORMIGRATE(ZSL%HYPRE_ZSL%X_H, HYPRE_MEMORY_HOST, HYPRE_IERR) + ENDIF +#endif + CALL HYPRE_IJVECTORGETVALUES(ZSL%HYPRE_ZSL%X_H, ZSL%NUNKH_LOCAL, ZSL%HYPRE_ZSL%INDICES, ZSL%X_H, HYPRE_IERR) #endif @@ -3908,6 +3973,17 @@ SUBROUTINE GET_H_MATRIX_LUDCMP STOP_STATUS = SETUP_STOP RETURN ENDIF + + ! Set execution policy +#ifdef WITH_HYPRE_DEVICE + IF (HYPRE_DEVICE_RUN) THEN + CALL HYPRE_SETEXECUTIONPOLICY(HYPRE_EXEC_DEVICE, IERR) + IF (MY_RANK==0) WRITE(LU_ERR,*) 'HYPRE : Setting execution policy to device (GPU)' + ELSE + CALL HYPRE_SETEXECUTIONPOLICY(HYPRE_EXEC_HOST, IERR) + IF (MY_RANK==0) WRITE(LU_ERR,*) 'HYPRE : Setting execution policy to host (CPU)' + ENDIF +#endif #endif END SELECT @@ -4050,10 +4126,10 @@ SUBROUTINE GET_H_MATRIX_LUDCMP IF (ALLOCATED(ZSL%HYPRE_ZSL%INDICES)) DEALLOCATE(ZSL%HYPRE_ZSL%INDICES) ALLOCATE( ZSL%HYPRE_ZSL%INDICES(MAX(1,ZSL%NUNKH_LOCAL)) ) - CALL HYPRE_IJMATRIXCREATE(MPI_COMM_WORLD,ZSL%LOWER_ROW-1,ZSL%UPPER_ROW-1,& - ZSL%LOWER_ROW-1,ZSL%UPPER_ROW-1,ZSL%HYPRE_ZSL%A_H,HYPRE_IERR) - CALL HYPRE_IJMATRIXSETOBJECTTYPE(ZSL%HYPRE_ZSL%A_H,HYPRE_PARCSR,HYPRE_IERR) - CALL HYPRE_IJMATRIXINITIALIZE(ZSL%HYPRE_ZSL%A_H,HYPRE_IERR) + CALL HYPRE_IJMATRIXCREATE(MPI_COMM_WORLD, ZSL%LOWER_ROW-1, ZSL%UPPER_ROW-1,& + ZSL%LOWER_ROW-1, ZSL%UPPER_ROW-1, ZSL%HYPRE_ZSL%A_H, HYPRE_IERR) + CALL HYPRE_IJMATRIXSETOBJECTTYPE(ZSL%HYPRE_ZSL%A_H, HYPRE_PARCSR, HYPRE_IERR) + CALL HYPRE_IJMATRIXINITIALIZE_V2(ZSL%HYPRE_ZSL%A_H, HYPRE_MEMORY_HOST, HYPRE_IERR) IF(ZSL%NUNKH_LOCAL > 0) THEN ZSL%JD_MAT_H = ZSL%JD_MAT_H - 1 DO IROW=1,ZSL%NUNKH_LOCAL @@ -4073,11 +4149,11 @@ SUBROUTINE GET_H_MATRIX_LUDCMP ! Create right hand side vector CALL HYPRE_IJVECTORCREATE(MPI_COMM_WORLD, ZSL%LOWER_ROW-1, ZSL%UPPER_ROW-1, ZSL%HYPRE_ZSL%F_H, HYPRE_IERR) CALL HYPRE_IJVECTORSETOBJECTTYPE(ZSL%HYPRE_ZSL%F_H, HYPRE_PARCSR, HYPRE_IERR) - CALL HYPRE_IJVECTORINITIALIZE(ZSL%HYPRE_ZSL%F_H, HYPRE_IERR) + CALL HYPRE_IJVECTORINITIALIZE_V2(ZSL%HYPRE_ZSL%F_H, HYPRE_MEMORY_HOST, HYPRE_IERR) ! Create solution vector CALL HYPRE_IJVECTORCREATE(MPI_COMM_WORLD, ZSL%LOWER_ROW-1, ZSL%UPPER_ROW-1, ZSL%HYPRE_ZSL%X_H, HYPRE_IERR) CALL HYPRE_IJVECTORSETOBJECTTYPE(ZSL%HYPRE_ZSL%X_H, HYPRE_PARCSR, HYPRE_IERR) - CALL HYPRE_IJVECTORINITIALIZE(ZSL%HYPRE_ZSL%X_H, HYPRE_IERR) + CALL HYPRE_IJVECTORINITIALIZE_V2(ZSL%HYPRE_ZSL%X_H, HYPRE_MEMORY_HOST, HYPRE_IERR) ! Set values CALL HYPRE_IJVECTORSETVALUES(ZSL%HYPRE_ZSL%F_H, ZSL%NUNKH_LOCAL, ZSL%HYPRE_ZSL%INDICES, ZSL%F_H, HYPRE_IERR) CALL HYPRE_IJVECTORSETVALUES(ZSL%HYPRE_ZSL%X_H, ZSL%NUNKH_LOCAL, ZSL%HYPRE_ZSL%INDICES, ZSL%X_H, HYPRE_IERR) @@ -4088,6 +4164,15 @@ SUBROUTINE GET_H_MATRIX_LUDCMP CALL HYPRE_IJVECTORGETOBJECT(ZSL%HYPRE_ZSL%F_H, ZSL%HYPRE_ZSL%PAR_F_H, HYPRE_IERR) CALL HYPRE_IJVECTORGETOBJECT(ZSL%HYPRE_ZSL%X_H, ZSL%HYPRE_ZSL%PAR_X_H, HYPRE_IERR) + ! Move data to GPU RAM +#ifdef WITH_HYPRE_DEVICE + IF (HYPRE_DEVICE_RUN) THEN + CALL HYPRE_IJMATRIXMIGRATE(ZSL%HYPRE_ZSL%A_H, HYPRE_MEMORY_DEVICE, HYPRE_IERR) + CALL HYPRE_IJVECTORMIGRATE(ZSL%HYPRE_ZSL%F_H, HYPRE_MEMORY_DEVICE, HYPRE_IERR) + CALL HYPRE_IJVECTORMIGRATE(ZSL%HYPRE_ZSL%X_H, HYPRE_MEMORY_DEVICE, HYPRE_IERR) + ENDIF +#endif + ! Create solver (Parallel Compressed Sparse Row Preconditioned Conjugate Gradient) CALL HYPRE_PARCSRPCGCREATE(MPI_COMM_WORLD, ZSL%HYPRE_ZSL%SOLVER, HYPRE_IERR) CALL HYPRE_PARCSRPCGSETMAXITER(ZSL%HYPRE_ZSL%SOLVER, HYPRE_SOLVER_MAXIT, HYPRE_IERR) @@ -4105,9 +4190,19 @@ SUBROUTINE GET_H_MATRIX_LUDCMP CALL HYPRE_BOOMERAMGSETTOL(ZSL%HYPRE_ZSL%PRECOND, HYPRE_PRECOND_TOL, HYPRE_IERR) CALL HYPRE_BOOMERAMGSETMAXITER(ZSL%HYPRE_ZSL%PRECOND, HYPRE_PRECOND_MAXITER, HYPRE_IERR) CALL HYPRE_PARCSRPCGSETPRECOND(ZSL%HYPRE_ZSL%SOLVER, HYPRE_PRECOND_ID, ZSL%HYPRE_ZSL%PRECOND, HYPRE_IERR) + ! Solver setup CALL HYPRE_PARCSRPCGSETUP(ZSL%HYPRE_ZSL%SOLVER, ZSL%HYPRE_ZSL%PARCSR_A_H,& ZSL%HYPRE_ZSL%PAR_F_H, ZSL%HYPRE_ZSL%PAR_X_H, HYPRE_IERR) + + ! Move vectors to GPU RAM +#ifdef WITH_HYPRE_DEVICE + IF (HYPRE_DEVICE_RUN) THEN + CALL HYPRE_IJVECTORMIGRATE(ZSL%HYPRE_ZSL%F_H, HYPRE_MEMORY_HOST, HYPRE_IERR) + CALL HYPRE_IJVECTORMIGRATE(ZSL%HYPRE_ZSL%X_H, HYPRE_MEMORY_HOST, HYPRE_IERR) + ENDIF +#endif + #endif END SELECT LIBRARY_SELECT diff --git a/Source/read.f90 b/Source/read.f90 index dc364bc2ebc..130286b3c62 100644 --- a/Source/read.f90 +++ b/Source/read.f90 @@ -4820,7 +4820,7 @@ SUBROUTINE READ_REAC ALLOCATE(RN%SPEC_ID_NU_READ(RN%N_SMIX)) RN%SPEC_ID_NU_READ = 'null' RN%SPEC_ID_NU_READ(1:RN%N_SMIX)=SPEC_ID_NU(1:RN%N_SMIX) - + ELSE SIMPLE_IF RN%N_SMIX = 3 RN%N_SPEC_READ = 0 @@ -6307,7 +6307,7 @@ SUBROUTINE PROC_PART ! Allocate necessary boundary information if BLOWING heat transfer correction is applied IF (SF%BLOWING) LPC%INCLUDE_BOUNDARY_PROP2_TYPE = .TRUE. - + ! If COLOR is not assigned to the PART class, use the SURF color if it has been specified IF (ALL(LPC%RGB==0) .AND. .NOT.ALL(SF%RGB==SURFACE(INERT_SURF_INDEX)%RGB)) LPC%RGB=SF%RGB @@ -9701,7 +9701,8 @@ SUBROUTINE READ_PRES NAMELIST /PRES/ BAROCLINIC,CHECK_POISSON,FISHPAK_BC,ITERATION_SUSPEND_FACTOR, & MAX_PRESSURE_ITERATIONS,PRESSURE_RELAX_TIME,PRESSURE_TOLERANCE, & - RELAXATION_FACTOR,SOLVER,SUSPEND_PRESSURE_ITERATIONS,TUNNEL_PRECONDITIONER,VELOCITY_TOLERANCE + RELAXATION_FACTOR,SOLVER,SUSPEND_PRESSURE_ITERATIONS,& + HYPRE_DEVICE_RUN,TUNNEL_PRECONDITIONER,VELOCITY_TOLERANCE ! Read the single PRES line @@ -12063,7 +12064,7 @@ SUBROUTINE READ_VENT VT%UNDIVIDED_INPUT_AREA = (XB_USER(2)-XB_USER(1))*(XB_USER(4)-XB_USER(3)) ! For circular vent account for VENT XB may only partially overlap circle - IF (RADIUS>0._EB) THEN + IF (RADIUS>0._EB) THEN IF (ABS(XB_USER(1)-XB_USER(2))<=SPACING(XB_USER(2))) VT%UNDIVIDED_INPUT_AREA = & CIRCLE_CELL_INTERSECTION_AREA(XYZ(2),XYZ(3),RADIUS,XB_USER(3),XB_USER(4),XB_USER(5),XB_USER(6)) IF (ABS(XB_USER(3)-XB_USER(4))<=SPACING(XB_USER(4))) VT%UNDIVIDED_INPUT_AREA = & diff --git a/Verification/Pressure_Solver/dancing_eddies_uglmat_hypre.fds b/Verification/Pressure_Solver/dancing_eddies_uglmat_hypre.fds index 6f8c26979b8..d9b6ca114fd 100644 --- a/Verification/Pressure_Solver/dancing_eddies_uglmat_hypre.fds +++ b/Verification/Pressure_Solver/dancing_eddies_uglmat_hypre.fds @@ -1,4 +1,4 @@ -&HEAD CHID='dancing_eddies_uglmat_hypre', TITLE='Simple 2D Tunnel test of UGLMAT HYPRE solver'/ +&HEAD CHID='dancing_eddies_uglmat_hypre', TITLE='Simple 2D Tunnel test of UGLMAT HYPRE solver'/ &MULT ID='mesh',DX=0.075,I_UPPER=3 / &MESH IJK=75,1,40, XB=0.0,0.075,-0.001,0.001,-0.02,0.020, MULT_ID='mesh' / @@ -9,7 +9,7 @@ &SPEC ID='LJ AIR', VISCOSITY=1.2E-5, BACKGROUND=.TRUE. / -&PRES CHECK_POISSON=T, SOLVER='UGLMAT HYPRE', MAX_PRESSURE_ITERATIONS=1 / +&PRES CHECK_POISSON=T, SOLVER='UGLMAT HYPRE', HYPRE_DEVICE_RUN=T, MAX_PRESSURE_ITERATIONS=1 / &SURF ID='WALL', FREE_SLIP=.TRUE., DEFAULT=.TRUE. / &SURF ID='PLATE', NO_SLIP=.TRUE. / @@ -32,5 +32,3 @@ &DEVC XYZ=0.,0.,0., QUANTITY='CPU TIME', ID='cpu' / &TAIL / - - diff --git a/Verification/Pressure_Solver/dancing_eddies_ulmat_hypre.fds b/Verification/Pressure_Solver/dancing_eddies_ulmat_hypre.fds index b8d6f5fbc7f..8219689e9e0 100644 --- a/Verification/Pressure_Solver/dancing_eddies_ulmat_hypre.fds +++ b/Verification/Pressure_Solver/dancing_eddies_ulmat_hypre.fds @@ -1,4 +1,4 @@ -&HEAD CHID='dancing_eddies_ulmat_hypre', TITLE='Simple 2D Tunnel, test of ULMAT with TUNNEL_PRECONDITIONER'/ +&HEAD CHID='dancing_eddies_ulmat_hypre', TITLE='Simple 2D Tunnel, test of ULMAT with TUNNEL_PRECONDITIONER'/ &MULT ID='mesh',DX=0.075,I_UPPER=3 / &MESH IJK=75,1,40, XB=0.0,0.075,-0.001,0.001,-0.02,0.020, MULT_ID='mesh' / @@ -7,7 +7,7 @@ &MISC SIMULATION_MODE='DNS', STRATIFICATION=.FALSE., NOISE=.FALSE. / -&PRES SOLVER='ULMAT HYPRE', CHECK_POISSON=T, TUNNEL_PRECONDITIONER=T / +&PRES SOLVER='ULMAT HYPRE', CHECK_POISSON=T, HYPRE_DEVICE_RUN=T, TUNNEL_PRECONDITIONER=T / &SPEC ID='LJ AIR', VISCOSITY=1.2E-5, BACKGROUND=.TRUE. / @@ -33,5 +33,3 @@ &DEVC XYZ=0.,0.,0., QUANTITY='CPU TIME', ID='cpu' / &TAIL / - - diff --git a/Verification/Pressure_Solver/duct_flow_uglmat_hypre.fds b/Verification/Pressure_Solver/duct_flow_uglmat_hypre.fds index 3e9dfa54fff..499c898ba54 100644 --- a/Verification/Pressure_Solver/duct_flow_uglmat_hypre.fds +++ b/Verification/Pressure_Solver/duct_flow_uglmat_hypre.fds @@ -11,7 +11,7 @@ MESH IJK=32,32,32, XB=-.2,6.2,-.2,6.2,-.2,6.2 / &TIME T_END=60.0 / &MISC STRATIFICATION=.FALSE. / -&PRES CHECK_POISSON=T, SOLVER='UGLMAT HYPRE' / +&PRES CHECK_POISSON=T, SOLVER='UGLMAT HYPRE', HYPRE_DEVICE_RUN=T / &SURF ID='DUCT', COLOR='GRAY 30', TRANSPARENCY=0.5 / diff --git a/Verification/Pressure_Solver/duct_flow_ulmat_hypre.fds b/Verification/Pressure_Solver/duct_flow_ulmat_hypre.fds index 42f8e6bb4e0..4991e125a1d 100644 --- a/Verification/Pressure_Solver/duct_flow_ulmat_hypre.fds +++ b/Verification/Pressure_Solver/duct_flow_ulmat_hypre.fds @@ -11,7 +11,7 @@ MESH IJK=32,32,32, XB=-.2,6.2,-.2,6.2,-.2,6.2 / &TIME T_END=60.0 / &MISC STRATIFICATION=.FALSE. / -&PRES CHECK_POISSON=T, SOLVER='ULMAT HYPRE', VELOCITY_TOLERANCE=0.001, MAX_PRESSURE_ITERATIONS=1000 / +&PRES CHECK_POISSON=T, SOLVER='ULMAT HYPRE', HYPRE_DEVICE_RUN=T, VELOCITY_TOLERANCE=0.001, MAX_PRESSURE_ITERATIONS=1000 / &SURF ID='DUCT', COLOR='GRAY 30', TRANSPARENCY=0.5 /