Conda installation of core library

CMakeLists.txt

@@ -21,6 +21,10 @@ list(APPEND CMAKE_INSTALL_RPATH $ENV{CONDA_PREFIX}/lib)
 list(APPEND CMAKE_PREFIX_PATH "$ENV{CONDA_PREFIX}")
 list(APPEND CMAKE_PREFIX_PATH "$ENV{CONDA_PREFIX}/Library")
 
-add_subdirectory(catkit_core)
+find_package(catkit2-core QUIET)
+if (NOT catkit2-core_FOUND)
+    message(STATUS "catkit2-core not found, building from source")
+    add_subdirectory(catkit2-core)
+endif()
+
 add_subdirectory(catkit2)
-add_subdirectory(benchmarks)

catkit2-core/CMakeLists.txt

@@ -0,0 +1,124 @@
+cmake_minimum_required(VERSION 3.21)
+
+project(catkit2_core)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED on)
+
+if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR})
+  message(FATAL_ERROR "In-source builds not allowed. Please make a new directory (called a build directory) and run CMake from there. You may need to remove CMakeCache.txt.")
+endif()
+
+add_compile_options($<$<CXX_COMPILER_ID:MSVC>:/MP1>)
+
+file(GLOB_RECURSE SOURCES "./src/*.cpp")
+file(GLOB_RECURSE HEADERS "./src/*.h")
+
+add_library(catkit2_core STATIC)
+target_sources(catkit2_core PRIVATE "${SOURCES}")
+target_include_directories(
+  catkit2_core
+  INTERFACE
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+  $<INSTALL_INTERFACE:include>
+)
+
+set_property(TARGET catkit2_core PROPERTY POSITION_INDEPENDENT_CODE ON)
+target_compile_definitions(catkit2_core PUBLIC PROTOBUF_USE_DLLS)
+
+if (MSVC)
+  # disable warning: 'identifier': class 'type' needs to have dll-interface to be used by clients of class 'type2'
+  target_compile_options(catkit2_core PUBLIC /wd4251)
+endif()
+
+# Link ZeroMQ
+find_package(ZeroMQ REQUIRED)
+target_include_directories(catkit2_core PUBLIC ${ZeroMQ_INCLUDE_DIR})
+target_link_libraries(catkit2_core PUBLIC libzmq)
+if (WIN32)
+  target_link_libraries(catkit2_core PUBLIC wsock32 ws2_32 Iphlpapi)
+else()
+  target_link_libraries(catkit2_core PUBLIC pthread)
+  if (NOT APPLE)
+    target_link_libraries(catkit2_core PUBLIC rt)
+  endif (NOT APPLE)
+endif (WIN32)
+
+# Add includes for cppzmq
+find_package(cppzmq REQUIRED)
+target_include_directories(catkit2_core PUBLIC ${CPPZMQ_INCLUDE_DIR})
+
+# Link Eigen
+find_package(Eigen3 REQUIRED NO_MODULE)
+target_link_libraries (catkit2_core PUBLIC Eigen3::Eigen)
+
+# Link nlohmann JSON
+find_package(nlohmann_json)
+target_include_directories(catkit2_core PRIVATE ${JSON_INCLUDE_DIR})
+
+# Link protobuf
+INCLUDE(FindProtobuf)
+find_package(Protobuf REQUIRED)
+target_include_directories(catkit2_core PUBLIC ${PROTOBUF_INCLUDE_DIR})
+target_link_libraries(catkit2_core PUBLIC ${PROTOBUF_LIBRARY})
+
+set(PROTO_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/src/")
+message("Proto binary dir: ${PROTO_BINARY_DIR}")
+file(MAKE_DIRECTORY "${PROTO_BINARY_DIR}")
+protobuf_generate(
+  TARGET catkit2_core
+  IMPORT_DIRS proto
+  PROTOC_OUT_DIR "${PROTO_BINARY_DIR}"
+  PROTOS
+    proto/core.proto
+    proto/logging.proto
+    proto/service.proto
+    proto/testbed.proto
+    proto/tracing.proto
+)
+target_include_directories(catkit2_core PUBLIC "$<BUILD_INTERFACE:${PROTO_BINARY_DIR}>")
+# Generate and install CatkitCoreConfig.cmake
+include(CMakePackageConfigHelpers)
+
+configure_package_config_file(
+  "${CMAKE_CURRENT_SOURCE_DIR}/cmake/CatkitCoreConfig.cmake.in"
+  "${CMAKE_CURRENT_BINARY_DIR}/CatkitCoreConfig.cmake"
+  INSTALL_DESTINATION lib/cmake/catkit2_core
+)
+
+install(FILES
+  "${CMAKE_CURRENT_BINARY_DIR}/CatkitCoreConfig.cmake"
+  DESTINATION "lib/cmake/catkit2_core"
+)
+
+install(
+  TARGETS catkit2_core
+  EXPORT CatkitCoreTargets
+  LIBRARY DESTINATION lib
+)
+
+# Generate and install CatkitCoreTargets.cmake
+install(
+  EXPORT CatkitCoreTargets
+  FILE CatkitCoreTargets.cmake
+  DESTINATION "lib/cmake/catkit2_core"
+)
+
+install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/src/"
+        DESTINATION include/catkit2_core
+        FILES_MATCHING
+        PATTERN "*.h"
+        PATTERN "*.inl"
+)
+install(DIRECTORY "${PROTO_BINARY_DIR}"
+        DESTINATION include/catkit2_core
+        FILES_MATCHING
+        PATTERN "*.h"
+)
+install(DIRECTORY "${PROTO_SOURCE_DIR}"
+        DESTINATION include/catkit2_core/proto
+        FILES_MATCHING
+        PATTERN "*.proto"
+)
+
+add_subdirectory(benchmarks)

benchmarks/CMakeLists.txt → catkit2-core/benchmarks/CMakeLists.txt

@@ -11,53 +11,63 @@ endif()
 
 # DataStream latency benchmark
 add_executable(datastream_latency datastream_latency.cpp)
-target_include_directories(datastream_latency PUBLIC ../catkit_core)
-target_link_libraries(datastream_latency PUBLIC catkit_core)
+target_include_directories(datastream_latency PUBLIC ../catkit2_core)
+target_link_libraries(datastream_latency PUBLIC catkit2_core)
 
 # Datastream submit benchmark
 add_executable(datastream_submit datastream_submit.cpp)
-target_include_directories(datastream_submit PUBLIC ../catkit_core)
-target_link_libraries(datastream_submit PUBLIC catkit_core)
+target_include_directories(datastream_submit PUBLIC ../catkit2_core)
+target_link_libraries(datastream_submit PUBLIC catkit2_core)
 
 # Timestamp benchmark
 add_executable(timestamp timestamp.cpp)
-target_include_directories(timestamp PUBLIC ../catkit_core)
-target_link_libraries(timestamp PUBLIC catkit_core)
+target_include_directories(timestamp PUBLIC ../catkit2_core)
+target_link_libraries(timestamp PUBLIC catkit2_core)
+
+# Free list allocator benchmark
+add_executable(free_list_allocator free_list_allocator.cpp)
+target_include_directories(free_list_allocator PUBLIC ../catkit2_core)
+target_link_libraries(free_list_allocator PUBLIC catkit2_core)
 
 # Pool allocator benchmark
 add_executable(pool_allocator pool_allocator.cpp)
-target_include_directories(pool_allocator PUBLIC ../catkit_core)
-target_link_libraries(pool_allocator PUBLIC catkit_core)
+target_include_directories(pool_allocator PUBLIC ../catkit2_core)
+target_link_libraries(pool_allocator PUBLIC catkit2_core)
 
 # Hash map benchmark
 add_executable(hash_map hash_map.cpp)
-target_include_directories(hash_map PUBLIC ../catkit_core)
-target_link_libraries(hash_map PUBLIC catkit_core)
+target_include_directories(hash_map PUBLIC ../catkit2_core)
+target_link_libraries(hash_map PUBLIC catkit2_core)
 
 # Uuid generator benchmark
 add_executable(uuid_generator uuid_generator.cpp)
-target_include_directories(uuid_generator PUBLIC ../catkit_core)
-target_link_libraries(uuid_generator PUBLIC catkit_core)
+target_include_directories(uuid_generator PUBLIC ../catkit2_core)
+target_link_libraries(uuid_generator PUBLIC catkit2_core)
 
 # MessageBroker benchmark
 add_executable(message_broker message_broker.cpp)
-target_include_directories(message_broker PUBLIC ../catkit_core)
-target_link_libraries(message_broker PUBLIC catkit_core)
+target_include_directories(message_broker PUBLIC ../catkit2_core)
+target_link_libraries(message_broker PUBLIC catkit2_core)
 
 # BuddyAllocator benchmark
 add_executable(buddy_allocator buddy_allocator.cpp)
-target_include_directories(buddy_allocator PUBLIC ../catkit_core)
-target_link_libraries(buddy_allocator PUBLIC catkit_core)
+target_include_directories(buddy_allocator PUBLIC ../catkit2_core)
+target_link_libraries(buddy_allocator PUBLIC catkit2_core)
 
 # HybridPoolAllocator benchmark
 add_executable(hybrid_pool_allocator hybrid_pool_allocator.cpp)
-target_include_directories(hybrid_pool_allocator PUBLIC ../catkit_core)
-target_link_libraries(hybrid_pool_allocator PUBLIC catkit_core)
+target_include_directories(hybrid_pool_allocator PUBLIC ../catkit2_core)
+target_link_libraries(hybrid_pool_allocator PUBLIC catkit2_core)
 
 # Event latency benchmark
 add_executable(event_latency event_latency.cpp)
-target_include_directories(event_latency PUBLIC ../catkit_core)
-target_link_libraries(event_latency PUBLIC catkit_core)
+target_include_directories(event_latency PUBLIC ../catkit2_core)
+target_link_libraries(event_latency PUBLIC catkit2_core)
+
+# Free list benchmark
+add_executable(free_list free_list_allocator.cpp)
+target_include_directories(free_list PUBLIC ../catkit2_core)
+target_link_libraries(free_list PUBLIC catkit2_core)
 
 # Add install files
 install(TARGETS datastream_latency DESTINATION bin)
@@ -70,3 +80,4 @@ install(TARGETS message_broker DESTINATION bin)
 install(TARGETS buddy_allocator DESTINATION bin)
 install(TARGETS hybrid_pool_allocator DESTINATION bin)
 install(TARGETS event_latency DESTINATION bin)
+install(TARGETS free_list DESTINATION bin)

catkit2-core/benchmarks/free_list_allocator.cpp

@@ -0,0 +1,145 @@
+#include "BuddyAllocator.h"
+#include "Timing.h"
+#include "LocalMemory.h"
+
+#include <iostream>
+
+void benchmark_linux_scalability()
+{
+	const size_t N = 10000000;
+	const size_t BLOCK_SIZE = 32;
+	const size_t MAX_SIZE = BLOCK_SIZE * (1 << 25);
+
+	auto *handles = new BuddyAllocator::Handle[N];
+
+	size_t buffer_size = BuddyAllocator::GetSharedStateSize(MAX_SIZE, BLOCK_SIZE);
+	auto buffer = LocalMemory::Create(buffer_size);
+
+	auto stream = StructStream(buffer);
+	auto allocator = BuddyAllocator::Create(stream, MAX_SIZE, BLOCK_SIZE);
+
+	auto start = GetTimeStamp();
+
+	for (size_t i = 0; i < N; ++i)
+	{
+		handles[i] = allocator->Allocate(16);
+	}
+
+	for (size_t i = 0; i < N; ++i)
+	{
+		allocator->Release(handles[i]);
+	}
+
+	auto end = GetTimeStamp();
+
+	std::cout << "Linux Scalability:" << std::endl;
+	std::cout << "Time: " << (end - start) / 1e9 << " sec" << std::endl;
+	std::cout << "Throughput: " << 2 * N / ((end - start) / 1e9) << " ops/s" << std::endl;
+    std::cout << "Time per operation: " << (end - start) / (2 * N) << " ns" << std::endl;
+
+	delete[] handles;
+}
+
+void benchmark_threadtest()
+{
+	const size_t N = 100;
+	const size_t M = 100000;
+	const size_t BLOCK_SIZE = 32;
+	const size_t DEPTH = 24;
+	const size_t MAX_SIZE = BLOCK_SIZE * (1 << DEPTH);
+
+	auto *handles = new BuddyAllocator::Handle[M];
+
+	size_t buffer_size = BuddyAllocator::GetSharedStateSize(MAX_SIZE, BLOCK_SIZE);
+	auto buffer = LocalMemory::Create(buffer_size);
+
+	auto stream = StructStream(buffer);
+	auto allocator = BuddyAllocator::Create(stream, MAX_SIZE, BLOCK_SIZE);
+
+	auto start = GetTimeStamp();
+
+	for (size_t i = 0; i < M; ++i)
+	{
+		for (size_t j = 0; j < N; ++j)
+		{
+			handles[j] = allocator->Allocate(16);
+		}
+
+		for (size_t j = 0; j < N; ++j)
+		{
+			allocator->Release(handles[j]);
+		}
+	}
+
+	auto end = GetTimeStamp();
+
+	std::cout << "Threadtest:" << std::endl;
+	std::cout << "Time: " << (end - start) / 1e9 << " sec" << std::endl;
+	std::cout << "Throughput: " << 2 * N * M / ((end - start) / 1e9) << " ops/s" << std::endl;
+    std::cout << "Time per operation: " << (end - start) / (2 * N * M) << " ns" << std::endl;
+
+	delete[] handles;
+}
+
+void benchmark_larson()
+{
+	const size_t ALIGNMENT = 32;
+
+	const size_t N = 10000000;
+	const size_t M = 1000;
+	const size_t MIN_SIZE = 1;
+	const size_t MAX_SIZE = 16;
+	const size_t BLOCK_SIZE = 16;
+	const size_t DEPTH = 20;
+	const size_t SIZE = BLOCK_SIZE * (1 << DEPTH);
+
+	auto *handles = new BuddyAllocator::Handle[M];
+	for (size_t i = 0; i < M; ++i)
+		handles[i] = BuddyAllocator::INVALID_HANDLE;
+
+	size_t buffer_size = BuddyAllocator::GetSharedStateSize(SIZE, BLOCK_SIZE);
+	auto buffer = LocalMemory::Create(buffer_size);
+
+	auto stream = StructStream(buffer);
+	auto allocator = BuddyAllocator::Create(stream, SIZE, BLOCK_SIZE);
+
+	auto *indices = new size_t[N];
+	auto *sizes = new size_t[N];
+	for (size_t i = 0; i < N; ++i)
+	{
+		indices[i] = rand() % M;
+		sizes[i] = (MIN_SIZE + (rand() % (MAX_SIZE - MIN_SIZE))) * BLOCK_SIZE;
+	}
+
+	auto start = GetTimeStamp();
+
+	for (size_t i = 0; i < N; ++i)
+	{
+		size_t index = indices[i];
+		size_t size = sizes[i];
+
+		if (handles[index] != BuddyAllocator::INVALID_HANDLE)
+		{
+			allocator->Release(handles[index]);
+		}
+
+		handles[index] = allocator->Allocate(size);
+	}
+
+	auto end = GetTimeStamp();
+	std::cout << "Larson benchmark:" << std::endl;
+	std::cout << "Time: " << (end - start) / 1e9 << " sec" << std::endl;
+	std::cout << "Throughput: " << (N * 2 - M) / ((end - start) / 1e9) << " ops/s" << std::endl;
+    std::cout << "Time per operation: " << (end - start) / (2 * N - M) << " ns" << std::endl;
+
+	delete[] handles;
+}
+
+int main(int argc, char **argv)
+{
+	benchmark_linux_scalability();
+	benchmark_threadtest();
+	benchmark_larson();
+
+	return 0;
+}

catkit2-core/recipe/bld.bat

@@ -0,0 +1,6 @@
+mkdir build
+cd build
+
+cmake .. -A x64
+cmake --build . --config Release
+cmake --install . --prefix %PREFIX%

catkit2-core/recipe/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+mkdir -p build
+cd build
+
+cmake .. -DCMAKE_BUILD_TYPE=Release
+cmake --build . --config Release -v
+cmake --install . --prefix=$PREFIX