AMREX_GPU_MAX_THREADS: 256 -> 128

Author: WeiqunZhang

Docs/sphinx_documentation/source/GPU.rst

@@ -229,9 +229,9 @@ Building with CMake
 
 To build AMReX with GPU support in CMake, add
 ``-DAMReX_GPU_BACKEND=CUDA|HIP|SYCL`` to the ``cmake`` invocation, for CUDA,
-HIP and SYCL, respectively. By default, AMReX uses 256 threads per GPU
+HIP and SYCL, respectively. By default, AMReX uses 128 threads per GPU
 block/group in most situations. This can be changed with
-``-DAMReX_GPU_MAX_THREADS=N``, where ``N`` is 128 for example.
+``-DAMReX_GPU_MAX_THREADS=N``, where ``N`` is 256 for example.
 
 Enabling CUDA support
 ^^^^^^^^^^^^^^^^^^^^^
@@ -1166,7 +1166,7 @@ GPU block size
 
 By default, :cpp:`ParallelFor` launches ``AMREX_GPU_MAX_THREADS`` threads
 per GPU block, where ``AMREX_GPU_MAX_THREADS`` is a compile-time constant
-with a default value of 256.  The users can also explicitly specify the
+with a default value of 128.  The users can also explicitly specify the
 number of threads per block by :cpp:`ParallelFor<MY_BLOCK_SIZE>(...)`, where
 ``MY_BLOCK_SIZE`` is a multiple of the warp size (e.g., 128).  This allows
 the users to do performance tuning for individual kernels.

Tools/CMake/AMReXOptions.cmake

@@ -133,7 +133,7 @@ if (NOT AMReX_GPU_BACKEND STREQUAL NONE)
    message( STATUS "   AMReX_GPU_BACKEND = ${AMReX_GPU_BACKEND}")
 
    # We might set different default for different GPUs in the future.
-   set(AMReX_GPU_MAX_THREADS_DEFAULT "256")
+   set(AMReX_GPU_MAX_THREADS_DEFAULT "128")
    set(AMReX_GPU_MAX_THREADS ${AMReX_GPU_MAX_THREADS_DEFAULT} CACHE STRING
        "Maximum number of GPU threads per block" )
    message( STATUS "   AMReX_GPU_MAX_THREADS = ${AMReX_GPU_MAX_THREADS}")

Tools/GNUMake/Make.defs

@@ -269,7 +269,7 @@ else
 endif
 
 # Maximum number of GPU threads per block.
-CUDA_MAX_THREADS ?= 256
+CUDA_MAX_THREADS ?= 128
 GPU_MAX_THREADS ?= $(CUDA_MAX_THREADS)
 
 ifeq ($(USE_CUDA),TRUE)