Merge pull request #9 from E3SM-Project/develop

Develop

Author: ambrad

CMakeLists.txt

@@ -7,11 +7,25 @@ function (prc var)
   message ("${var}: ${${var}}")
 endfunction ()
 
+function (list2str list str)
+  string (REPLACE ";" " " tmp "${list}")
+  set (${str} ${tmp} PARENT_SCOPE)
+endfunction ()
+
 set (COMPOSE_DEBUG_MPI FALSE CACHE LOGICAL "If true, insert debugging code into MPI wrappers.")
+set (COMPOSE_MIMIC_GPU FALSE CACHE LOGICAL "If true, use non-optimal OpenMP threading to debug GPU-like parallelism.")
+set (COMPOSE_QLT_TIME  FALSE CACHE LOGICAL "If true, time QLT operations and display timers at the end.")
 
 if (Kokkos_DIR)
   include (${Kokkos_DIR}/kokkos_generated_settings.cmake)
   set (Kokkos_INCLUDE ${Kokkos_DIR}/include)
+  list2str ("${KOKKOS_CXXFLAGS}" Kokkos_CXX_FLAGS)
+  list2str ("${KOKKOS_LD_FLAGS}" Kokkos_LD_FLAGS)
+  string (FIND "${KOKKOS_GMAKE_DEVICES}" "Cuda" cuda_str_pos)
+  set (CUDA_BUILD FALSE)
+  if (${cuda_str_pos} GREATER -1)
+    set (CUDA_BUILD TRUE)
+  endif ()
 else ()
   message (FATAL_ERROR "COMPOSE requires Kokkos_DIR")
 endif ()
@@ -65,12 +79,16 @@ if ("${cmake_build_type_str}" STREQUAL "debug")
 endif ()
 
 if (DEBUG_BUILD)
-  message ("Enable COMPOSE_DEBUG_MPI because build type is DEBUG.")
+  message ("Enable COMPOSE_DEBUG_MPI and COMPOSE_MIMIC_GPU because build type is DEBUG.")
   set (COMPOSE_DEBUG_MPI TRUE)
+  set (COMPOSE_MIMIC_GPU TRUE)
 endif ()
 
-set (COMPOSE_COMPILE_FLAGS "-g ${KOKKOS_LINK_FLAGS} ${CMAKE_CXX_FLAGS}")
-set (COMPOSE_LINK_FLAGS "-L${Kokkos_DIR}/lib ${KOKKOS_LINK_FLAGS}")
+set (COMPOSE_COMPILE_FLAGS "-g ${Kokkos_CXX_FLAGS} ${CMAKE_CXX_FLAGS}")
+if (CUDA_BUILD)
+  set (COMPOSE_COMPILE_FLAGS "${COMPOSE_COMPILE_FLAGS} --expt-relaxed-constexpr")
+endif ()
+set (COMPOSE_LINK_FLAGS "-L${Kokkos_DIR}/lib ${Kokkos_LD_FLAGS}")
 set (COMPOSE_INCLUDES ${Kokkos_INCLUDE} ${CMAKE_CURRENT_SOURCE_DIR}/share ${CMAKE_BINARY_DIR}/config)
 set (COMPOSE_LIBRARIES ${KOKKOS_LIBS_LIST})
 
@@ -85,8 +103,8 @@ configure_file (${CMAKE_CURRENT_SOURCE_DIR}/compose_config.h.in ${CMAKE_BINARY_D
 
 add_library (${PROJECT_NAME} ${SOURCES})
 set_target_properties (${PROJECT_NAME} PROPERTIES
-  COMPILE_FLAGS ${COMPOSE_COMPILE_FLAGS}
-  LINK_FLAGS ${COMPOSE_LINK_FLAGS})
+  COMPILE_FLAGS "${COMPOSE_COMPILE_FLAGS}"
+  LINK_FLAGS "${COMPOSE_LINK_FLAGS}")
 target_include_directories (${PROJECT_NAME} PUBLIC cedr siqk)
 target_include_directories (${PROJECT_NAME} PRIVATE siqk cedr)
 target_include_directories (${PROJECT_NAME} PUBLIC ${COMPOSE_INCLUDES})

LICENSE

@@ -0,0 +1,28 @@
+COMPOSE version 1.0: Copyright 2018 National Technology & Engineering Solutions
+of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the
+U.S. Government retains certain rights in this software.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,40 @@
+## COMPOSE: Compact Multi-moment Performance-Portable Semi-Lagrangian methods
+
+COMPOSE provides libraries for semi-Lagrangian transport and, together or
+separately, property preservation:
+
+* CEDR: Communication-Efficient Constrained Density Reconstructors.
+* SIQK: Spherical Polygon Intersection and Quadrature.
+
+# Building and installing
+
+First, install [Kokkos](https://github.com/kokkos/kokkos).
+For example, in a typical environment using OpenMP, run:
+```
+    git clone https://github.com/kokkos/kokkos.git
+    ./kokkos/generate_makefile.bash --with-serial --with-openmp --prefix=/path/to/my/kokkos/install --compiler=g++
+    make -j8 install
+```
+
+Second, configure, build, and test COMPOSE:
+```
+    cmake \
+        -D Kokkos_DIR=/path/to/my/kokkos/install \
+        -D CMAKE_INSTALL_PREFIX=/path/to/my/compose/install \
+        /path/to/compose/repo
+    make -j8
+    ctest
+```
+
+Optionally, third, install COMPOSE:
+```
+    make install
+```
+
+# Licence
+
+COMPOSE version 1.0: Copyright 2018 National Technology & Engineering Solutions
+of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the
+U.S. Government retains certain rights in this software.
+
+This software is released under the BSD licence; see [LICENSE](./LICENSE).

README.txt

@@ -1,13 +1,16 @@
-add_executable (cedr_test cedr_test.cpp)
-set_target_properties (cedr_test PROPERTIES
-  COMPILE_FLAGS ${COMPOSE_COMPILE_FLAGS}
-  LINK_FLAGS ${COMPOSE_LINK_FLAGS})
-
-target_include_directories (cedr_test PRIVATE ${COMPOSE_INCLUDES})
-target_link_libraries (cedr_test ${PROJECT_NAME} ${COMPOSE_LIBRARIES})
+foreach (exe cedr_test cedr_test_mpi_device_ptr)
+  add_executable (${exe} ${exe}.cpp)
+  set_target_properties (${exe} PROPERTIES
+    COMPILE_FLAGS ${COMPOSE_COMPILE_FLAGS}
+    LINK_FLAGS ${COMPOSE_LINK_FLAGS})
+  target_include_directories (${exe} PRIVATE ${COMPOSE_INCLUDES})
+  target_link_libraries (${exe} ${PROJECT_NAME} ${COMPOSE_LIBRARIES})
+endforeach ()
 
 add_test (NAME cedr-test-unit
   COMMAND $<TARGET_FILE:cedr_test> -t)
+add_test (NAME cedr-test-mpi-device-ptr
+  COMMAND $<TARGET_FILE:cedr_test_mpi_device_ptr>)
 add_test (NAME cedr-test-unit-mpi
   COMMAND ${COMPOSE_TEST_MPIRUN} ${COMPOSE_TEST_MPIFLAGS} -np ${COMPOSE_TEST_NRANK}
   $<TARGET_FILE:cedr_test> -t --proc-random -nc 111 -nt 11)

cedr/CMakeLists.txt

@@ -1,3 +1,6 @@
+// COMPOSE version 1.0: Copyright 2018 NTESS. This software is released under
+// the BSD license; see LICENSE in the top-level directory.
+
 #ifndef INCLUDE_CEDR_HPP
 #define INCLUDE_CEDR_HPP