From ac903fecf2a98e71ebb430a1ff71461af774e534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicente=20Mataix=20Ferr=C3=A1ndiz?= Date: Tue, 18 Feb 2025 11:44:16 +0100 Subject: [PATCH 1/4] [Build] Add support for benchmarking with Google Benchmark library --- CMakeLists.txt | 78 ++++++++++++++++++++++++++++++++++----- README.md | 1 + benchmarks/CMakeLists.txt | 18 +++++++++ 3 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 benchmarks/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 1cf86ae2..20fe8aa4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,7 @@ OPTION ( CO_SIM_IO_BUILD_PYTHON "Building the CoSimIO for Python" OPTION ( CO_SIM_IO_BUILD_FORTRAN "Building the CoSimIO for Fortran" OFF ) OPTION ( CO_SIM_IO_STRICT_COMPILER "Compiler has more warnings" OFF ) OPTION ( CO_SIM_IO_BUILD_TESTING "Build tests" ${BUILD_TESTING} ) +OPTION ( CO_SIM_IO_BUILD_BENCHMARK "Build benchmarks" OFF ) if(NOT DEFINED CO_SIM_IO_BUILD_TYPE) if(CMAKE_BUILD_TYPE) @@ -44,9 +45,10 @@ message(" CO_SIM_IO_BUILD_C: " ${CO_SIM_IO_BUILD_C}) message(" CO_SIM_IO_BUILD_PYTHON: " ${CO_SIM_IO_BUILD_PYTHON}) message(" CO_SIM_IO_BUILD_FORTRAN: " ${CO_SIM_IO_BUILD_FORTRAN}) message(" CO_SIM_IO_STRICT_COMPILER: " ${CO_SIM_IO_STRICT_COMPILER}) +message(" CO_SIM_IO_BUILD_BENCHMARK: " ${CO_SIM_IO_BUILD_BENCHMARK}) message("") -# needs to be here as otherwise doesn't find MPI properly +# Needs to be here as otherwise doesn't find MPI properly if (CO_SIM_IO_BUILD_MPI) find_package(MPI REQUIRED) add_definitions( -DCO_SIM_IO_USING_MPI ) @@ -56,6 +58,7 @@ if (CO_SIM_IO_BUILD_MPI) include_directories(SYSTEM ${MPI_INCLUDE_PATH}) endif() +# Threads are needed for the CoSimIO library find_package (Threads REQUIRED) # detect whether CoSimIO is integrated into another project @@ -70,8 +73,9 @@ if (NOT hasParent) else(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) message("-- User defined install dir ${CMAKE_INSTALL_PREFIX}") endif(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) -endif() +endif(NOT hasParent) +# If the compiler is MSVC, we need to set the correct flags if(MSVC) # /Zc:__cplusplus: required such that "__cplusplus" is set to the correct value # see https://devblogs.microsoft.com/cppblog/msvc-now-correctly-reports-__cplusplus/ @@ -126,14 +130,15 @@ endif() message("CMAKE_CXX_FLAGS:" ${CMAKE_CXX_FLAGS}) message("CMAKE_C_FLAGS:" ${CMAKE_C_FLAGS}) +# In case of MacOS, we need to set CMP0042 to NEW to avoid warnings if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") cmake_policy(SET CMP0042 NEW) -endif() +endif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") -# clean the bin directory +# Clean the bin directory if (NOT hasParent) file(REMOVE_RECURSE bin) -endif() +endif(NOT hasParent) # Adding CoSimIO library file(GLOB_RECURSE co_sim_io_source_files @@ -149,6 +154,7 @@ generate_export_header( co_sim_io EXPORT_MACRO_NAME CO_SIM_IO_API EXPORT_FILE_NA install(TARGETS co_sim_io DESTINATION bin) +# MPI communication if (CO_SIM_IO_BUILD_MPI) # optionally enable communication via MPI OPTION ( CO_SIM_IO_BUILD_MPI_COMMUNICATION "Enabling communication via MPI" OFF ) @@ -167,20 +173,23 @@ if (CO_SIM_IO_BUILD_MPI) target_link_libraries(co_sim_io_mpi co_sim_io ${MPI_LIBRARIES}) install(TARGETS co_sim_io_mpi DESTINATION bin) -endif() +endif(CO_SIM_IO_BUILD_MPI) target_include_directories(co_sim_io PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/co_sim_io) target_include_directories(co_sim_io SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/external_libraries) target_include_directories(co_sim_io SYSTEM PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/external_libraries/asio/include) +# C bindings if (CO_SIM_IO_BUILD_C) add_subdirectory(co_sim_io/c) -endif() +endif(CO_SIM_IO_BUILD_C) +# Python bindings if (CO_SIM_IO_BUILD_PYTHON) add_subdirectory(co_sim_io/python) -endif() +endif(CO_SIM_IO_BUILD_PYTHON) +# Fortran bindings if (CO_SIM_IO_BUILD_FORTRAN) if (NOT CO_SIM_IO_BUILD_C) message(FATAL_ERROR "Fortran requires the C interface!") @@ -190,8 +199,9 @@ if (CO_SIM_IO_BUILD_FORTRAN) co_sim_io/fortran NO_EXTERNAL_INSTALL ) -endif() +endif(CO_SIM_IO_BUILD_FORTRAN) +# Testing if (CO_SIM_IO_BUILD_TESTING) if(CMAKE_MAJOR_VERSION LESS 3) message(FATAL_ERROR "Building the tests requires CMake 3") @@ -203,4 +213,52 @@ if (CO_SIM_IO_BUILD_TESTING) "tests/compiled_config.json.in" "${CMAKE_CURRENT_SOURCE_DIR}/tests/compiled_config.json" ) -endif() +endif(CO_SIM_IO_BUILD_TESTING) + +# Benchmarking +if(CO_SIM_IO_BUILD_BENCHMARK MATCHES ON) + # Includes for the Google Benchmark library + # This function automatically configures a given application to build its benchmarks + macro(co_sim_io_add_benchmarks) + set(options USE_MPI USE_CUSTOM_MAIN) + set(oneValueArgs TARGET WORKING_DIRECTORY) + set(multiValueArgs SOURCES) + + cmake_parse_arguments(CO_SIM_IO_ADD_BENCHMARK "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if(CO_SIM_IO_ADD_BENCHMARK_SOURCES) + include(GoogleBenchmark) + endif(CO_SIM_IO_ADD_BENCHMARK_SOURCES) + endmacro(co_sim_io_add_benchmarks) + + # Add the definitions if required + ADD_DEFINITIONS(-DCO_SIM_IO_BUILD_BENCHMARKING) + + # Retrieve a copy of the current directory's `COMPILE_OPTIONS` + get_directory_property(co_sim_io_root_compile_options COMPILE_OPTIONS) + + # Disable warnings (needed by centos. We should all love centos, it clearly needs some affection) + add_compile_options(-w) + + # Fetch the Google Benchmark library + include(FetchContent) + FetchContent_Declare( + googlebenchmark + URL https://github.com/google/benchmark/archive/v1.9.0.zip + ) + # For Windows: Prevent overriding the parent project's compiler/linker settings + set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) + set(CMAKE_INSTALL_LIBDIR "bin") + set(CMAKE_INSTALL_BINDIR "bin") + FetchContent_GetProperties(googlebenchmark) + if(NOT googlebenchmark_POPULATED) + FetchContent_Populate(googlebenchmark) + add_subdirectory(${googlebenchmark_SOURCE_DIR} ${googlebenchmark_BINARY_DIR} EXCLUDE_FROM_ALL) + endif(NOT googlebenchmark_POPULATED) + + # Restore the current directory's old `COMPILE_OPTIONS` + set_directory_properties(PROPERTIES COMPILE_OPTIONS "${co_sim_io_root_compile_options}") + + # Add the benchmarking directory + add_subdirectory(benchmarks) +endif(CO_SIM_IO_BUILD_BENCHMARK MATCHES ON) \ No newline at end of file diff --git a/README.md b/README.md index a01708c4..46e6a1d3 100644 --- a/README.md +++ b/README.md @@ -62,3 +62,4 @@ Coupling requires frequent exchange of data. Therefore the _CoSimIO_ uses the me - [filesystem](https://github.com/gulrak/filesystem) Header-only single-file std::filesystem compatible helper library, based on the C++17 specs - [asio](https://think-async.com/Asio/) for socket based interprocess communication - [doctest](https://github.com/onqtam/doctest) C++ testing framework for the unit tests +- [Google Benchmark](https://github.com/google/benchmark) for microbenchmarking and performance testing \ No newline at end of file diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt new file mode 100644 index 00000000..0c3c211d --- /dev/null +++ b/benchmarks/CMakeLists.txt @@ -0,0 +1,18 @@ +include_directories( ${CMAKE_SOURCE_DIR}/co_sim_io ) + +# CoSimIO benchmark sources. Disabled by default +if(${CO_SIM_IO_BUILD_BENCHMARK} MATCHES ON) + file(GLOB_RECURSE CO_SIM_IO_BENCHMARK_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ) + + foreach(file ${CO_SIM_IO_BENCHMARK_SOURCES}) + get_filename_component(filename ${file} NAME_WE) + add_executable(${filename} ${file}) + target_link_libraries(${filename} PUBLIC co_sim_io benchmark::benchmark) + set_target_properties(${filename} PROPERTIES COMPILE_DEFINITIONS "CO_SIM_IO_BENCHMARK=IMPORT,API") + install(TARGETS ${filename} DESTINATION "bin/benchmarks") + endforeach(file ${CO_SIM_IO_BENCHMARK_SOURCES}) +endif(${CO_SIM_IO_BUILD_BENCHMARK} MATCHES ON) + +# TODO: Add MPI support in the future if required \ No newline at end of file From 8feb23aef3a97a43979fd87d6205cbcb3c3a6bf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicente=20Mataix=20Ferr=C3=A1ndiz?= Date: Tue, 18 Feb 2025 11:44:24 +0100 Subject: [PATCH 2/4] [Benchmark] Add Google Benchmark for import/export of large data in CoSimIO --- benchmarks/communication_benchmark.cpp | 124 +++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 benchmarks/communication_benchmark.cpp diff --git a/benchmarks/communication_benchmark.cpp b/benchmarks/communication_benchmark.cpp new file mode 100644 index 00000000..729b517e --- /dev/null +++ b/benchmarks/communication_benchmark.cpp @@ -0,0 +1,124 @@ +// ______ _____ _ ________ +// / ____/___ / ___/(_)___ ___ / _/ __ | +// / / / __ \\__ \/ / __ `__ \ / // / / / +// / /___/ /_/ /__/ / / / / / / // // /_/ / +// \____/\____/____/_/_/ /_/ /_/___/\____/ +// Kratos CoSimulationApplication +// +// License: BSD License, see license.txt +// +// Main authors: Vicente Mataix Ferrandiz + +// System includes +#include +#include +#include +#include +#include + +// External includes +#include + +// Project includes +#include "includes/communication/communication.hpp" +#include "includes/communication/factory.hpp" + +namespace CoSimIO +{ + +//-------------------------------------------------------------------------- +// Helper: ExportDataHelper from tests/co_sim_io/cpp/test_communication.cpp +//-------------------------------------------------------------------------- +void ExportDataHelper( + Info settings, + const std::vector>& DataToExport) +{ + settings.Set("my_name", "thread"); + settings.Set("connect_to", "main"); + settings.Set("is_primary_connection", false); + settings.Set("echo_level", 0); + + using Communication = Internals::Communication; + std::unique_ptr p_comm = + Internals::CommunicationFactory().Create( + settings, std::make_shared()); + + // Give the primary thread a chance to prepare (as in your test) + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + + Info connect_info; + p_comm->Connect(connect_info); + + Info export_info; + export_info.Set("identifier", "data_exchange"); + + for (const auto& data : DataToExport) { + Internals::DataContainerStdVectorReadOnly data_container(data); + p_comm->ExportData(export_info, data_container); + } + + Info disconnect_info; + p_comm->Disconnect(disconnect_info); +} + +//-------------------------------------------------------------------------- +// Google Benchmark for "import_export_large_data" from tests/co_sim_io/cpp/test_communication.cpp +//-------------------------------------------------------------------------- +static void BM_ImportExportLargeData(benchmark::State& state) { + // Get size in MB from benchmark parameter. + const std::size_t size_MB = static_cast(state.range(0)); + const std::size_t size_B = size_MB * 1024 * 1024; + const std::size_t size_vec = size_B / sizeof(double); + + // Prepare the large data vector (size_MB MB of doubles) + std::vector> exp_data(1); + exp_data[0].resize(size_vec); + std::iota(exp_data[0].begin(), exp_data[0].end(), 0); + + // Set up common communication settings + Info settings; + settings.Set("communication_format", "pipe"); + // You can add more settings if needed + + // Benchmark loop: each iteration performs a full connect-import-disconnect cycle + for (auto _ : state) { + using Communication = Internals::Communication; + std::unique_ptr main_comm = + Internals::CommunicationFactory().Create( + settings, std::make_shared()); + + // Launch the exporter in a separate thread (it uses ExportDataHelper) + std::thread export_thread(ExportDataHelper, settings, exp_data); + + // Connect the main (import) side + Info connect_info; + main_comm->Connect(connect_info); + + // Prepare a container for the imported data + std::vector imported_data; + Internals::DataContainerStdVector data_container(imported_data); + + Info import_info; + import_info.Set("identifier", "data_exchange"); + + // Perform the import operation (this is what we want to measure) + main_comm->ImportData(import_info, data_container); + + // Validate that the imported data has the expected size. + assert(data_container.GetData().size() == exp_data[0].size()); + + // Disconnect the main communication + Info disconnect_info; + main_comm->Disconnect(disconnect_info); + + // Ensure the exporter thread completes before starting the next iteration + export_thread.join(); + } +} + +// Register the benchmark with different sizes in MB (e.g., 5 MB, 10 MB, 20 MB) +BENCHMARK(BM_ImportExportLargeData)->Arg(5)->Arg(10)->Arg(20); + +} // namespace CoSimIO + +BENCHMARK_MAIN(); \ No newline at end of file From 7d7037ce3c3732b0a7c1fa5093bd667cfdba9226 Mon Sep 17 00:00:00 2001 From: Vicente Mataix Ferrandiz Date: Tue, 18 Feb 2025 13:39:02 +0100 Subject: [PATCH 3/4] [Benchmark] Remove data size validation from import/export benchmark --- benchmarks/communication_benchmark.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/benchmarks/communication_benchmark.cpp b/benchmarks/communication_benchmark.cpp index 729b517e..dc454938 100644 --- a/benchmarks/communication_benchmark.cpp +++ b/benchmarks/communication_benchmark.cpp @@ -104,9 +104,6 @@ static void BM_ImportExportLargeData(benchmark::State& state) { // Perform the import operation (this is what we want to measure) main_comm->ImportData(import_info, data_container); - // Validate that the imported data has the expected size. - assert(data_container.GetData().size() == exp_data[0].size()); - // Disconnect the main communication Info disconnect_info; main_comm->Disconnect(disconnect_info); From 4f54b2ade8f833f0b3a6fdb03b0a12ef902940a7 Mon Sep 17 00:00:00 2001 From: Vicente Mataix Ferrandiz Date: Tue, 18 Feb 2025 14:28:44 +0100 Subject: [PATCH 4/4] Refactor communication benchmark for improved readability and update argument sizes --- benchmarks/communication_benchmark.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/benchmarks/communication_benchmark.cpp b/benchmarks/communication_benchmark.cpp index dc454938..0727975b 100644 --- a/benchmarks/communication_benchmark.cpp +++ b/benchmarks/communication_benchmark.cpp @@ -39,9 +39,7 @@ void ExportDataHelper( settings.Set("echo_level", 0); using Communication = Internals::Communication; - std::unique_ptr p_comm = - Internals::CommunicationFactory().Create( - settings, std::make_shared()); + std::unique_ptr p_comm = Internals::CommunicationFactory().Create(settings, std::make_shared()); // Give the primary thread a chance to prepare (as in your test) std::this_thread::sleep_for(std::chrono::milliseconds(20)); @@ -53,7 +51,7 @@ void ExportDataHelper( export_info.Set("identifier", "data_exchange"); for (const auto& data : DataToExport) { - Internals::DataContainerStdVectorReadOnly data_container(data); + const Internals::DataContainerStdVectorReadOnly data_container(data); p_comm->ExportData(export_info, data_container); } @@ -77,15 +75,16 @@ static void BM_ImportExportLargeData(benchmark::State& state) { // Set up common communication settings Info settings; + settings.Set("my_name", "main"); + settings.Set("connect_to", "thread"); settings.Set("communication_format", "pipe"); - // You can add more settings if needed + settings.Set("is_primary_connection", true); + settings.Set("echo_level", 0); // Benchmark loop: each iteration performs a full connect-import-disconnect cycle for (auto _ : state) { using Communication = Internals::Communication; - std::unique_ptr main_comm = - Internals::CommunicationFactory().Create( - settings, std::make_shared()); + std::unique_ptr main_comm = Internals::CommunicationFactory().Create(settings, std::make_shared()); // Launch the exporter in a separate thread (it uses ExportDataHelper) std::thread export_thread(ExportDataHelper, settings, exp_data); @@ -113,8 +112,8 @@ static void BM_ImportExportLargeData(benchmark::State& state) { } } -// Register the benchmark with different sizes in MB (e.g., 5 MB, 10 MB, 20 MB) -BENCHMARK(BM_ImportExportLargeData)->Arg(5)->Arg(10)->Arg(20); +// Register the benchmark with different sizes in MB (e.g., 5 MB, 20 MB, 100 MB and 2000 MB) +BENCHMARK(BM_ImportExportLargeData)->Arg(5)->Arg(20)->Arg(100)->Arg(2000); } // namespace CoSimIO