Add benchmarks to CI test build and analysis

Author: sliwowitz

script/run_generate.sh

@@ -86,7 +86,7 @@ mkdir -p build/
 cd build/
 
 "${ALPAKA_CI_CMAKE_EXECUTABLE}" --log-level=VERBOSE -G "${ALPAKA_CI_CMAKE_GENERATOR}" ${ALPAKA_CI_CMAKE_GENERATOR_PLATFORM}\
-    -Dalpaka_BUILD_EXAMPLES=ON -DBUILD_TESTING=ON \
+    -Dalpaka_BUILD_EXAMPLES=ON -Dalpaka_BUILD_BENCHMARK=ON -DBUILD_TESTING=ON \
     "$(env2cmake BOOST_ROOT)" -DBOOST_LIBRARYDIR="${ALPAKA_CI_BOOST_LIB_DIR}/lib" -DBoost_USE_STATIC_LIBS=ON -DBoost_USE_MULTITHREADED=ON -DBoost_USE_STATIC_RUNTIME=OFF -DBoost_ARCHITECTURE="-x64" \
     "$(env2cmake CMAKE_BUILD_TYPE)" "$(env2cmake CMAKE_CXX_FLAGS)" "$(env2cmake CMAKE_C_COMPILER)" "$(env2cmake CMAKE_CXX_COMPILER)" "$(env2cmake CMAKE_EXE_LINKER_FLAGS)" "$(env2cmake CMAKE_CXX_EXTENSIONS)"\
     "$(env2cmake alpaka_ACC_CPU_B_SEQ_T_SEQ_ENABLE)" "$(env2cmake alpaka_ACC_CPU_B_SEQ_T_THREADS_ENABLE)" "$(env2cmake alpaka_ACC_CPU_B_SEQ_T_FIBERS_ENABLE)" \

test/analysis/headerCheck/CMakeLists.txt

@@ -49,6 +49,8 @@ if(alpaka_CHECK_HEADERS)
         PRIVATE common)
 
     set_target_properties(headerCheckTest PROPERTIES FOLDER "test/analysis")
+    # Catch2 benchmark macros must be defined, otherwise the benchmarking headers will not pass the check.
+    target_compile_definitions(${_TARGET_NAME} PUBLIC CATCH_CONFIG_ENABLE_BENCHMARKING)	 
 
     add_test(NAME ${_TARGET_NAME} COMMAND ${_TARGET_NAME} ${_alpaka_TEST_OPTIONS})
 

test/benchmark/rand/CMakeLists.txt

@@ -23,7 +23,11 @@ set_target_properties(${_TARGET_NAME} PROPERTIES FOLDER "test/benchmark")
 target_compile_definitions(${_TARGET_NAME} PUBLIC CATCH_CONFIG_ENABLE_BENCHMARKING)
 
 if(alpaka_CI)
-	add_test(NAME ${_TARGET_NAME} COMMAND ${_TARGET_NAME} ${_alpaka_TEST_OPTIONS} --benchmark-samples 1)    
+	# For non-benchmarking CI test runs - It will only run the benchmark once to see if it works at all.
+	add_test(NAME ${_TARGET_NAME} COMMAND ${_TARGET_NAME} ${_alpaka_TEST_OPTIONS} --benchmark-samples 1)
+  	# Real automated benchmark runs will need to collect more samples (the default 100 is fine). The CI will then 
+  	# have to set another variable to indicate if it is only testing, or if it wants to do a full benchmark. 
 else()
+	# For full benchmark run - will collect 100 samples wor godd benchmark statistics.
 	add_test(NAME ${_TARGET_NAME} COMMAND ${_TARGET_NAME} ${_alpaka_TEST_OPTIONS})
 endif()

test/benchmark/rand/src/randBenchmark.cpp

@@ -62,7 +62,11 @@ TEST_CASE("defaultRandomGeneratorBenchmark", "[randBenchmark]")
     const Idx numThreads = std::thread::hardware_concurrency(); // TODO: GPU?
     std::cout << "Hardware threads: " << numThreads << std::endl;
 
-    const unsigned numPoints = GENERATE(100'000u, 1'000'000u, 10'000'000u, 100'000'000u, 1'000'000'000u);
+#ifdef ALPAKA_CI // Reduced benchmark set for automated test runs.
+    const unsigned numPoints = GENERATE(10u, 1'000'000u);
+#else
+    const unsigned numPoints = GENERATE(10u, 100'000u, 1'000'000u, 10'000'000u, 100'000'000u, 1'000'000'000u);
+#endif
 
     WorkDiv workdiv{alpaka::getValidWorkDiv<Acc>(
         devAcc,