a0x8o · a0x8o · Nov 26, 2024 · Jun 24, 2022 · Sep 1, 2022 · Dec 22, 2022
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -51,7 +51,7 @@ endif (__GIT_EXECUTABLE)
 # This must be set because version tags
 set(HYDROGEN_VERSION_MAJOR 1)
 set(HYDROGEN_VERSION_MINOR 5)
-set(HYDROGEN_VERSION_PATCH 2)
+set(HYDROGEN_VERSION_PATCH 4)
 set(HYDROGEN_VERSION_MAJOR_MINOR
     "${HYDROGEN_VERSION_MAJOR}.${HYDROGEN_VERSION_MINOR}")
 set(HYDROGEN_VERSION
@@ -101,10 +101,7 @@ if (NOT DEFINED CMAKE_POSITION_INDEPENDENT_CODE)
 endif ()
 
 # Feature-related options
-
-option(Hydrogen_ENABLE_ALUMINUM
-  "Enable the Aluminum package for improved device-side communication."
-  OFF)
+include(CMakeDependentOption)
 
 option(Hydrogen_ENABLE_CUDA
   "Search for CUDA support and enable related features if found."
@@ -114,6 +111,12 @@ option(Hydrogen_ENABLE_ROCM
   "Search for ROCm/HIP support and enable related features if found."
   OFF)
 
+cmake_dependent_option(Hydrogen_ENABLE_ROCTRACER
+  "Search for Roctracer and enable related features if found."
+  OFF
+  "Hydrogen_ENABLE_ROCM"
+  OFF)
+
 if (Hydrogen_ENABLE_ROCM AND Hydrogen_ENABLE_CUDA)
   message(FATAL_ERROR
     "ROCm and CUDA code paths are mutually exclusive. "
@@ -313,10 +316,6 @@ if (Hydrogen_ENABLE_CUDA)
     set(HYDROGEN_GPU_USE_TENSOR_OP_MATH TRUE)
   endif ()
 
-  if (Hydrogen_ENABLE_GPU_FP16)
-    set(HYDROGEN_GPU_USE_FP16 TRUE)
-  endif ()
-
   if (Hydrogen_ENABLE_CUB)
     if (CUDAToolkit_VERSION_MAJOR LESS 11)
       find_package(CUB MODULE REQUIRED)
@@ -358,74 +357,102 @@ if (Hydrogen_ENABLE_ROCM)
   find_package(rocsolver CONFIG REQUIRED)
   find_package(rocthrust CONFIG REQUIRED)
 
+  if (Hydrogen_ENABLE_ROCTRACER)
+    find_package(Roctracer MODULE COMPONENTS roctx)
+    set(HYDROGEN_HAVE_ROCTRACER ${Roctracer_FOUND})
+  endif ()
+
+  include(HydrogenCleanupHIPTargets)
+  h_clean_hip_targets()
+
   set(HYDROGEN_HAVE_ROCM TRUE)
   message(STATUS "Found ROCm/HIP toolchain. Using HIP/ROCm.")
 
   set(H_ROCM_CXX_LIBS
     hip::host
-    hip::hipcub
     roc::rocblas
     roc::rocsolver
-    roc::rocthrust)
-
+    roc::rocthrust
+    ${Roctracer_LIBRARIES})
+  if (HYDROGEN_HAVE_CUB)
+    list(APPEND H_ROCM_CXX_LIBS hip::hipcub)
+  endif ()
   set(H_ROCM_HIP_LIBS
     hip::device)
 endif (Hydrogen_ENABLE_ROCM)
 
 if (HYDROGEN_HAVE_CUDA OR HYDROGEN_HAVE_ROCM)
   set(HYDROGEN_HAVE_GPU TRUE)
+
+  if (Hydrogen_ENABLE_GPU_FP16)
+    set(HYDROGEN_GPU_USE_FP16 TRUE)
+  endif ()
 endif ()
 
-if (Hydrogen_ENABLE_ALUMINUM)
-  find_package(Aluminum 1.0.0 CONFIG QUIET)
+find_package(Aluminum 1.0.0 CONFIG QUIET)
+if (NOT Aluminum_FOUND AND Aluminum_NOT_FOUND_MESSAGE)
+  message(STATUS
+    "A candidate Aluminum > v1.0.0 was found, but was not selected:")
+  message(STATUS
+    "  ${Aluminum_NOT_FOUND_MESSAGE}")
+endif ()
+# Try again, since we're technically ok with >v0.7.0
+if (NOT Aluminum_FOUND)
+  find_package(Aluminum 0.7.0 CONFIG QUIET)
   if (NOT Aluminum_FOUND AND Aluminum_NOT_FOUND_MESSAGE)
     message(STATUS
-      "A candidate Aluminum > v1.0.0 was found, but was not selected:")
+      "A candidate Aluminum > v0.7.0 was found, but was not selected:")
     message(STATUS
       "  ${Aluminum_NOT_FOUND_MESSAGE}")
   endif ()
-  # Try again, since we're technically ok with >v0.7.0
-  if (NOT Aluminum_FOUND)
-    find_package(Aluminum 0.7.0 CONFIG QUIET)
-    if (NOT Aluminum_FOUND AND Aluminum_NOT_FOUND_MESSAGE)
-      message(STATUS
-        "A candidate Aluminum > v0.7.0 was found, but was not selected:")
-      message(STATUS
-        "  ${Aluminum_NOT_FOUND_MESSAGE}")
-    endif ()
-  endif ()
+endif ()
 
-  if (Aluminum_FOUND)
-    set(HYDROGEN_HAVE_ALUMINUM TRUE)
-    message(STATUS
-      "Found Aluminum@${ALUMINUM_VERSION}: ${Aluminum_DIR}")
+if (Aluminum_FOUND)
+  set(HYDROGEN_HAVE_ALUMINUM TRUE)
+  message(STATUS
+    "Found Aluminum@${ALUMINUM_VERSION}: ${Aluminum_DIR}")
 
-    if (HYDROGEN_HAVE_GPU AND AL_HAS_NCCL)
-      set(HYDROGEN_HAVE_NCCL2 TRUE)
-      message(STATUS "Aluminum detected with NCCL2 backend support.")
-    else ()
-      set(HYDROGEN_HAVE_NCCL2 FALSE)
-    endif (HYDROGEN_HAVE_GPU AND AL_HAS_NCCL)
+  if (HYDROGEN_HAVE_GPU AND AL_HAS_NCCL)
+    set(HYDROGEN_HAVE_NCCL2 TRUE)
+    message(STATUS "Aluminum detected with NCCL2 backend support.")
+  else ()
+    set(HYDROGEN_HAVE_NCCL2 FALSE)
+  endif (HYDROGEN_HAVE_GPU AND AL_HAS_NCCL)
 
-    if (HYDROGEN_HAVE_GPU AND AL_HAS_HOST_TRANSFER)
-      set(HYDROGEN_HAVE_AL_HOST_XFER TRUE)
-      message(STATUS "Aluminum detected with HostTransfer backend support.")
-    else ()
-      set(HYDROGEN_HAVE_AL_HOST_XFER FALSE)
-    endif (HYDROGEN_HAVE_GPU AND AL_HAS_HOST_TRANSFER)
+  if (HYDROGEN_HAVE_GPU AND AL_HAS_HOST_TRANSFER)
+    set(HYDROGEN_HAVE_AL_HOST_XFER TRUE)
+    message(STATUS "Aluminum detected with HostTransfer backend support.")
+  else ()
+    set(HYDROGEN_HAVE_AL_HOST_XFER FALSE)
+  endif (HYDROGEN_HAVE_GPU AND AL_HAS_HOST_TRANSFER)
 
-    if (HYDROGEN_HAVE_GPU AND AL_HAS_MPI_CUDA)
-      set(HYDROGEN_HAVE_AL_MPI_CUDA TRUE)
-      message(STATUS "Aluminum detected with MPI-CUDA backend support.")
-    else ()
-      set(HYDROGEN_HAVE_AL_MPI_CUDA FALSE)
-    endif (HYDROGEN_HAVE_GPU AND AL_HAS_MPI_CUDA)
+  if (HYDROGEN_HAVE_GPU AND AL_HAS_MPI_CUDA)
+    set(HYDROGEN_HAVE_AL_MPI_CUDA TRUE)
+    message(STATUS "Aluminum detected with MPI-CUDA backend support.")
+  else ()
+    set(HYDROGEN_HAVE_AL_MPI_CUDA FALSE)
+  endif (HYDROGEN_HAVE_GPU AND AL_HAS_MPI_CUDA)
+
+  # Check for in-place SendRecv.
+  if (ALUMINUM_VERSION VERSION_GREATER_EQUAL "1.3.0")
+    set(HYDROGEN_AL_SUPPORTS_INPLACE_SENDRECV TRUE)
   else ()
-    message(FATAL_ERROR "Aluminum support requested but not found. "
-      "Please set Aluminum_DIR to point to the installation prefix "
-      "for Aluminum.")
-  endif (Aluminum_FOUND)
-endif (Hydrogen_ENABLE_ALUMINUM)
+    set(HYDROGEN_AL_SUPPORTS_INPLACE_SENDRECV FALSE)
+  endif ()
+
+  if (HYDROGEN_AL_SUPPORTS_INPLACE_SENDRECV)
+    message(STATUS "Aluminum detected with in-place SendRecv support.")
+  else ()
+    message(STATUS "Aluminum detected WITHOUT in-place SendRecv support.")
+  endif ()
+
+else ()
+
+  message(FATAL_ERROR "Aluminum support required but not found. "
+    "Please set Aluminum_ROOT to its installation prefix or add "
+    "the installation prefix to CMAKE_PREFIX_PATH.")
+
+endif (Aluminum_FOUND)
 
 # Sets up EL_RESTRICT and EL_HAVE_PRETTY_FUNCTION
 include(detect/CXX)
@@ -484,7 +511,7 @@ add_subdirectory(src)
 # docs (which has the advantage that preprocessing will take
 # "{,hydrogen_}config.h" into consideration).
 configure_file("${PROJECT_SOURCE_DIR}/cmake/configure_files/config.h.in"
-  "${PROJECT_BINARY_DIR}/include/El/config.h")
+  "${PROJECT_BINARY_DIR}/include/El/config.h" ESCAPE_QUOTES)
 configure_file("${PROJECT_SOURCE_DIR}/cmake/configure_files/hydrogen_config.h.in"
   "${PROJECT_BINARY_DIR}/include/El/hydrogen_config.h")
 configure_file("${PROJECT_SOURCE_DIR}/doxy/Doxyfile.in"

diff --git a/ElementalREADME.md b/ElementalREADME.md
@@ -1,10 +1,3 @@
-<p align="left" style="padding: 20px">
-<img src="http://libelemental.org/_static/elemental.png">
-</p>
-
-[![Build Status](https://api.travis-ci.org/elemental/Elemental.svg?branch=master)](https://travis-ci.org/elemental/Elemental)
-[![Join the chat at https://gitter.im/elemental/chat](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/elemental/chat?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-
 **Elemental** is a modern C++ library for distributed-memory dense and
 sparse-direct linear algebra, conic optimization, and lattice reduction.
 The library was initially released in
@@ -15,7 +8,7 @@ was originally released during a project on [Parallel Sweeping Preconditioners](
 
 ### Documentation
 
-The (now outdated) [documentation for Elemental](http://libelemental.org/documentation) is built using [Sphinx](http://sphinx.pocoo.org) and the [Read the Docs Theme](http://docs.readthedocs.org/en/latest/theme.html)
+The (now outdated) documentation for Elemental is built using [Sphinx](http://sphinx.pocoo.org) and the [Read the Docs Theme](http://docs.readthedocs.org/en/latest/theme.html)
 
 ### Unique features
 

diff --git a/cmake/configure_files/HydrogenConfig.cmake.in b/cmake/configure_files/HydrogenConfig.cmake.in
@@ -92,6 +92,7 @@ if (_HYDROGEN_HAVE_CUDA)
 endif ()
 
 set(_HYDROGEN_HAVE_ROCM @HYDROGEN_HAVE_ROCM@)
+set(_HYDROGEN_HAVE_ROCTRACER @HYDROGEN_HAVE_ROCTRACER@)
 if (_HYDROGEN_HAVE_ROCM)
 
   find_dependency(hip CONFIG)
@@ -106,6 +107,14 @@ if (_HYDROGEN_HAVE_ROCM)
   find_dependency(rocblas CONFIG)
   find_dependency(rocsolver CONFIG)
   find_dependency(rocthrust CONFIG)
+
+  if (_HYDROGEN_HAVE_ROCTRACER)
+    find_dependency(Roctracer MODULE COMPONENTS roctx)
+  endif ()
+
+  include(HydrogenCleanupHIPTargets)
+  h_clean_hip_targets()
+
   set(HYDROGEN_HAVE_ROCM TRUE)
 endif (_HYDROGEN_HAVE_ROCM)
 
@@ -114,7 +123,7 @@ if (HYDROGEN_HAVE_HALF)
   find_dependency(HALF)
 endif ()
 
-if (_HYDROGEN_HAVE_CUDA)
+if (_HYDROGEN_HAVE_CUDA OR _HYDROGEN_HAVE_ROCM)
   set(HYDROGEN_GPU_USE_FP16 @HYDROGEN_GPU_USE_FP16@)
 endif ()
 

diff --git a/cmake/configure_files/hydrogen_config.h.in b/cmake/configure_files/hydrogen_config.h.in
@@ -40,6 +40,7 @@
 
 // ROCm stuff
 #cmakedefine HYDROGEN_HAVE_ROCM
+#cmakedefine HYDROGEN_HAVE_ROCTRACER
 
 // General GPU stuff
 #cmakedefine HYDROGEN_HAVE_CUB
@@ -51,6 +52,7 @@
 #cmakedefine HYDROGEN_HAVE_NCCL2
 #cmakedefine HYDROGEN_HAVE_AL_MPI_CUDA
 #cmakedefine HYDROGEN_HAVE_AL_HOST_XFER
+#cmakedefine HYDROGEN_AL_SUPPORTS_INPLACE_SENDRECV
 
 #cmakedefine HYDROGEN_ENSURE_HOST_MPI_BUFFERS
 #cmakedefine HYDROGEN_HAVE_CUDA_AWARE_MPI

diff --git a/cmake/modules/FindRoctracer.cmake b/cmake/modules/FindRoctracer.cmake
@@ -0,0 +1,78 @@
+# Sets the following variables
+#
+#   Roctracer_FOUND
+#   Roctracer_LIBRARIES
+#
+# Defines the following imported target:
+#
+#   roctracer::roctracer
+#   roctracer::roctracer_api
+#   roctracer::roctx_api
+#
+
+set(_supported_components roctracer roctx)
+if (NOT Roctracer_FIND_COMPONENTS)
+  set(Roctracer_FIND_COMPONENTS ${_supported_components})
+endif ()
+
+foreach (comp IN LISTS Roctracer_FIND_COMPONENTS)
+  if (NOT ${comp} IN_LIST _supported_components)
+    message(FATAL_ERROR
+      "Cannot specify component \"${comp}\" for package Roctracer. "
+      "Supported components are: ${_supported_components}.")
+  endif ()
+
+  set(_header_name "${comp}.h")
+  set(_lib_name "${comp}64")
+
+  find_path(${comp}_INCLUDE_PATH ${_header_name}
+    HINTS ${ROCM_PATH}/roctracer $ENV{ROCM_PATH}/roctracer
+    PATH_SUFFIXES include
+    DOC "The ${comp} include directory for roctracer."
+    NO_DEFAULT_PATH)
+  find_path(${comp}_INCLUDE_PATH ${_header_name}
+    HINTS ${ROCM_PATH}/include/roctracer $ENV{ROCM_PATH}/include/roctracer
+    DOC "The ${comp} include directory for roctracer."
+    NO_DEFAULT_PATH)
+  find_path(${comp}_INCLUDE_PATH ${_header_name})
+
+  find_library(${comp}_LIBRARY ${_lib_name}
+    HINTS ${ROCM_PATH}/roctracer $ENV{ROCM_PATH}/roctracer
+    HINTS ${ROCM_PATH} $ENV{ROCM_PATH}
+    PATH_SUFFIXES lib64 lib
+    DOC "The ${comp} library for roctracer."
+    NO_DEFAULT_PATH)
+  find_library(${comp}_LIBRARY ${_lib_name})
+
+  if (${comp}_LIBRARY AND ${comp}_INCLUDE_PATH)
+    set(Roctracer_${comp}_FOUND TRUE)
+
+    if (NOT TARGET roctracer::${comp}_api)
+      add_library(roctracer::${comp}_api INTERFACE IMPORTED)
+    endif ()
+    target_link_libraries(roctracer::${comp}_api INTERFACE
+      "${${comp}_LIBRARY}")
+    target_include_directories(roctracer::${comp}_api INTERFACE
+      "${${comp}_INCLUDE_PATH}")
+
+    mark_as_advanced(${comp}_LIBRARY)
+    mark_as_advanced(${comp}_INCLUDE_PATH)
+
+    list(APPEND _imported_libraries roctracer::${comp}_api)
+  else ()
+    set(Roctracer_${comp}_FOUND FALSE)
+  endif ()
+endforeach ()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Roctracer HANDLE_COMPONENTS)
+
+if (Roctracer_FOUND)
+  if (NOT TARGET roctracer::roctracer)
+    add_library(roctracer::roctracer INTERFACE IMPORTED)
+  endif ()
+  foreach (lib IN LISTS _imported_libraries)
+    target_link_libraries(roctracer::roctracer INTERFACE ${lib})
+  endforeach ()
+  set(Roctracer_LIBRARIES roctracer::roctracer)
+endif (Roctracer_FOUND)