diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 3bbf02e6f..7aa78096d 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -35,7 +35,7 @@ jobs: sudo apt upgrade -y sudo DEBIAN_FRONTEND=noninteractive sudo apt install -y tzdata # DEBIAN_FRONTEND=noninteractive apt install -y tzdata - sudo apt install software-properties-common -y + # sudo apt install software-properties-common -y sudo apt install cmake clang curl pkg-config -y - name: Make scripts executable @@ -65,7 +65,7 @@ jobs: path: capnp key: ${{ runner.os }}-cache-capnp - - name: Download and Compile capnp is not cached + - name: Download and Compile capnp if not cached if: steps.cache-capnp.outputs.cache-hit != 'true' run: | sudo ./scripts/download_install_dependencies.sh capnp compile @@ -78,7 +78,7 @@ jobs: run: | cmake -S . -B build -DCMAKE_CXX_COMPILER=$(which clang++) -DCMAKE_BUILD_TYPE=Debug cd build && make - ./scaler/object_storage/tests/test_update_record + ctest - name: Install Python Dependent Packages run: | diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index a77328518..e75ce8c8e 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -9,8 +9,12 @@ permissions: jobs: deploy: - runs-on: ubuntu-latest + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} environment: release + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] permissions: id-token: write steps: @@ -25,15 +29,36 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.8" + - name: Install dependencies run: | python -m pip install --upgrade pip pip install build pip install uv + pip install cibuildwheel==2.23.3 uv pip install --system -r pyproject.toml - name: Build package - run: python -m build + # run: python -m build + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_BEFORE_ALL: | + echo "Building deps" + yum install sudo -y; + sudo ./scripts/download_install_dependencies.sh capnp compile + sudo ./scripts/download_install_dependencies.sh capnp install + sudo ./scripts/download_install_dependencies.sh boost compile + sudo ./scripts/download_install_dependencies.sh boost install + CIBW_BUILD: "*manylinux_x86_64" + CIBW_SKIP: "pp*" + CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" + + - name: Build Sdist + run: | + python -m build --sdist + sudo mv dist/scaler* wheelhouse/. - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@db8f07d3871a0a180efa06b95d467625c19d5d5f # release/v1 + # path: ./wheelhouse/*.whl + path: ./wheelhouse/* diff --git a/CMakeLists.txt b/CMakeLists.txt index 7efb3ecc0..9968a989c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_CXX_STANDARD_REQUIRED ON) # set(CMAKE_CXX_SCAN_FOR_MODULES ON) set(CMAKE_CXX_SCAN_FOR_MODULES OFF) +set(BUILD_SHARED_LIBS ON CACHE BOOL "Build shared libraries") if(NOT SKBUILD_PROJECT_NAME) set(SKBUILD_PROJECT_NAME "scaler") @@ -36,5 +37,15 @@ else() message(STATUS "${PROJECT_NAME} ${CMAKE_BUILD_TYPE} build") endif() +find_package(CapnProto CONFIG REQUIRED) +get_target_property(CAPNP_INCLUDE_DIRS CapnProto::capnp INTERFACE_INCLUDE_DIRECTORIES) +message(STATUS "Found Capnp in ${CAPNP_INCLUDE_DIRS}") +# Make LSP happy +include_directories(${CAPNP_INCLUDE_DIRS}) +include_directories(${PROJECT_SOURCE_DIR}) add_subdirectory(scaler) + +if(NOT SKBUILD_STATE) + add_subdirectory(tests) +endif() diff --git a/scaler/CMakeLists.txt b/scaler/CMakeLists.txt index 6f3ced95e..ca45022f6 100644 --- a/scaler/CMakeLists.txt +++ b/scaler/CMakeLists.txt @@ -1,11 +1,2 @@ -set(BUILD_SHARED_LIBS ON CACHE BOOL "Build shared libraries") - -find_package(CapnProto CONFIG REQUIRED) - -get_target_property(CAPNP_INCLUDE_DIRS CapnProto::capnp INTERFACE_INCLUDE_DIRECTORIES) -message(STATUS "Found Capnp in ${CAPNP_INCLUDE_DIRS}") -# Make LSP happy -include_directories(${CAPNP_INCLUDE_DIRS}) - add_subdirectory(object_storage) diff --git a/scaler/object_storage/CMakeLists.txt b/scaler/object_storage/CMakeLists.txt index 9683047f9..87b5fa260 100644 --- a/scaler/object_storage/CMakeLists.txt +++ b/scaler/object_storage/CMakeLists.txt @@ -20,10 +20,3 @@ target_include_directories(server PRIVATE ${CMAKE_BINARY_DIR}) install(TARGETS server LIBRARY DESTINATION ${CMAKE_SOURCE_DIR}/scaler/lib) -message(STATUS ${CMAKE_SOURCE_DIR}/scaler/lib) - -# Build tests only when triggering manually for now. Later we change -# when we have proper unit tests -if(NOT SKBUILD_STATE) - add_subdirectory(tests) -endif() diff --git a/scaler/object_storage/io_helper.cpp b/scaler/object_storage/io_helper.cpp index e8dfdf2ba..ce418945e 100644 --- a/scaler/object_storage/io_helper.cpp +++ b/scaler/object_storage/io_helper.cpp @@ -15,9 +15,9 @@ #include #include -#include "constants.h" -#include "defs.h" #include "protocol/object_storage.capnp.h" +#include "scaler/object_storage/constants.h" +#include "scaler/object_storage/defs.h" using boost::asio::awaitable; using boost::asio::use_awaitable; diff --git a/scaler/object_storage/io_helper.h b/scaler/object_storage/io_helper.h index 10164b088..a95dc8fc9 100644 --- a/scaler/object_storage/io_helper.h +++ b/scaler/object_storage/io_helper.h @@ -5,7 +5,7 @@ #include #include -#include "defs.h" +#include "scaler/object_storage/defs.h" namespace scaler { namespace object_storage { diff --git a/scaler/object_storage/tests/CMakeLists.txt b/scaler/object_storage/tests/CMakeLists.txt deleted file mode 100644 index 11edd5ab6..000000000 --- a/scaler/object_storage/tests/CMakeLists.txt +++ /dev/null @@ -1,69 +0,0 @@ -include(FetchContent) -FetchContent_Declare( - googletest - URL https://github.com/google/googletest/archive/155b337c938a2953e5675f9dc18c99f05f4c85d0.zip -) -# For Windows: Prevent overriding the parent project's compiler/linker settings -set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) - -set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) -set(BUILD_GMOCK OFF CACHE BOOL "" FORCE) -set(BUILD_GTEST ON CACHE BOOL "" FORCE) -FetchContent_MakeAvailable(googletest) - -include(FetchContent) -set(ARGPARSE_GIT_REPOSITORY "https://github.com/p-ranav/argparse.git") -set(ARGPARSE_GIT_TAG "d924b84eba1f0f0adf38b20b7b4829f6f65b6570") -FetchContent_Declare( - argparse - GIT_REPOSITORY ${ARGPARSE_GIT_REPOSITORY} - GIT_TAG ${ARGPARSE_GIT_TAG} -) -FetchContent_MakeAvailable(argparse) -message(STATUS "Fetched argparse from ${ARGPARSE_GIT_REPOSITORY} @ ${ARGPARSE_GIT_TAG}") - -add_executable( - test_scaler_object - test_scaler_object.cpp - ../io_helper.cpp -) - -target_link_libraries( - test_scaler_object PRIVATE - GTest::gtest_main -) - -target_sources(test_scaler_object PRIVATE ${objectStorageSources}) -target_link_libraries(test_scaler_object PRIVATE CapnProto::capnp) -target_include_directories(test_scaler_object PRIVATE ${CMAKE_BINARY_DIR}) - -add_executable( - test_update_record - test_update_record.cpp - ../io_helper.cpp -) - -target_link_libraries( - test_update_record PRIVATE - GTest::gtest_main -) - -target_sources(test_update_record PRIVATE ${objectStorageSources}) -target_link_libraries(test_update_record PRIVATE CapnProto::capnp) -target_include_directories(test_update_record PRIVATE ${CMAKE_BINARY_DIR}) - -include(GoogleTest) -gtest_discover_tests(test_scaler_object) - -gtest_discover_tests(test_update_record) - -add_executable(test_server_main - test_server_main.cpp - ../io_helper.cpp -) - -target_include_directories(test_server_main PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/..) -target_sources(test_server_main PRIVATE ${objectStorageSources}) -target_link_libraries(test_server_main PRIVATE CapnProto::capnp) -target_include_directories(test_server_main PRIVATE ${CMAKE_BINARY_DIR}) -target_link_libraries(test_server_main PRIVATE argparse) diff --git a/scaler/object_storage/tests/test_server_main.cpp b/scaler/object_storage/tests/test_server_main.cpp deleted file mode 100644 index 60fd821d7..000000000 --- a/scaler/object_storage/tests/test_server_main.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../constants.h" -#include "../io_helper.h" -#include "../object_storage_server.h" -#include "argparse/argparse.hpp" -#include "version.h" - -// Helper macros to stringify the macro value -#define STRINGIFY_HELPER(x) #x -#define STRINGIFY(x) STRINGIFY_HELPER(x) - -using boost::asio::awaitable; -using boost::asio::co_spawn; -using boost::asio::detached; -using boost::asio::use_awaitable; -using boost::asio::ip::tcp; -namespace this_coro = boost::asio::this_coro; - -#if defined(BOOST_ASIO_ENABLE_HANDLER_TRACKING) -#define use_awaitable boost::asio::use_awaitable_t(__FILE__, __LINE__, __PRETTY_FUNCTION__) -#endif - -scaler::object_storage::ObjectStorageServer server; - -awaitable listener(boost::asio::ip::tcp::endpoint endpoint) { - auto executor = co_await this_coro::executor; - tcp::acceptor acceptor(executor, endpoint); - for (;;) { - tcp::socket socket = co_await acceptor.async_accept(use_awaitable); - co_spawn(executor, server.process_request(std::move(socket)), detached); - } -} - -int main(int argc, char* argv[]) { - std::string name = scaler::object_storage::DEFAULT_ADDR; - std::string port = scaler::object_storage::DEFAULT_PORT; - - argparse::ArgumentParser argParser("server", STRINGIFY(VERSION)); - argParser.add_argument("-n", "--name").default_value(name).help("Name to resolve, e.g., localhost, 127.0.0.1"); - argParser.add_argument("-p", "--port").default_value(port).help("Specify port to listen on"); - try { - argParser.parse_args(argc, argv); - } catch (const std::exception& err) { - std::cerr << err.what() << std::endl; - std::cerr << argParser; - std::exit(1); - } - - name = argParser.get("--name"); - port = argParser.get("--port"); - - try { - boost::asio::io_context io_context(1); - tcp::resolver resolver(io_context); - auto res = resolver.resolve(name, port); - - boost::asio::signal_set signals(io_context, SIGINT, SIGTERM); - signals.async_wait([&](auto, auto) { io_context.stop(); }); - - co_spawn(io_context, listener(res.begin()->endpoint()), detached); - - io_context.run(); - } catch (std::exception& e) { std::printf("Exception: %s\n", e.what()); } -} diff --git a/scripts/download_install_dependencies.sh b/scripts/download_install_dependencies.sh index 0b108a998..63e78508e 100755 --- a/scripts/download_install_dependencies.sh +++ b/scripts/download_install_dependencies.sh @@ -14,8 +14,8 @@ if [ "$1" == "boost" ]; then tar -xzf ${BOOST_PACKAGE_NAME} mv ${BOOST_FOLDER_NAME} boost elif [ "$2" == "install" ]; then - sudo cp -r boost/boost /usr/include/. - echo "Installed Boost into /usr/include/boost" + sudo cp -r boost/boost /usr/local/include/. + echo "Installed Boost into /usr/local/include/boost" else echo "Argument needs to be either compile or install" exit 1 @@ -29,12 +29,12 @@ elif [ "$1" == "capnp" ]; then tar -xzf ${CAPNP_PACKAGE_NAME} mv ${CAPNP_FOLDER_NAME} capnp cd capnp - ./configure --prefix=/usr/ + ./configure --prefix=/usr/local/ make -j6 check elif [ "$2" == "install" ]; then cd capnp sudo make install - echo "Installed capnp into /usr" + echo "Installed capnp into /usr/local" else echo "Argument needs to be either compile or install" exit 1 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 000000000..2d416b04c --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(object_storage) diff --git a/tests/object_storage/CMakeLists.txt b/tests/object_storage/CMakeLists.txt new file mode 100644 index 000000000..ff73612dd --- /dev/null +++ b/tests/object_storage/CMakeLists.txt @@ -0,0 +1,50 @@ +include(FetchContent) +FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/155b337c938a2953e5675f9dc18c99f05f4c85d0.zip +) +# For Windows: Prevent overriding the parent project's compiler/linker settings +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + +set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) +set(BUILD_GMOCK OFF CACHE BOOL "" FORCE) +set(BUILD_GTEST ON CACHE BOOL "" FORCE) +FetchContent_MakeAvailable(googletest) + +add_executable( + test_update_record + test_update_record.cpp + ${PROJECT_SOURCE_DIR}/scaler/object_storage/io_helper.cpp +) + +target_include_directories(test_update_record + PRIVATE ${CMAKE_BINARY_DIR} +) + +target_link_libraries( + test_update_record + PRIVATE GTest::gtest_main + PRIVATE CapnProto::capnp +) + +add_test(NAME test_update_record COMMAND test_update_record) + +add_executable( + test_scaler_object + test_scaler_object.cpp + ${PROJECT_SOURCE_DIR}/scaler/object_storage/io_helper.cpp + ${PROJECT_SOURCE_DIR}/scaler/object_storage/server.cpp + ${CMAKE_BINARY_DIR}/protocol/object_storage.capnp.c++ +) + +target_include_directories(test_scaler_object + PRIVATE ${CMAKE_BINARY_DIR} +) + +target_link_libraries( + test_scaler_object + PRIVATE GTest::gtest_main + PRIVATE CapnProto::capnp +) + +add_test(NAME test_scaler_object COMMAND test_scaler_object) diff --git a/scaler/object_storage/tests/test_scaler_object.cpp b/tests/object_storage/test_scaler_object.cpp similarity index 94% rename from scaler/object_storage/tests/test_scaler_object.cpp rename to tests/object_storage/test_scaler_object.cpp index a8e74bbbe..270d73ae3 100644 --- a/scaler/object_storage/tests/test_scaler_object.cpp +++ b/tests/object_storage/test_scaler_object.cpp @@ -9,10 +9,13 @@ #include #include #include +#include +#include -#include "../constants.h" -#include "../defs.h" #include "protocol/object_storage.capnp.h" +#include "scaler/object_storage/constants.h" +#include "scaler/object_storage/defs.h" +#include "scaler/object_storage/server.h" using boost::asio::buffer; using boost::asio::ip::tcp; @@ -22,6 +25,21 @@ using boost::asio::awaitable; using scaler::object_storage::CAPNP_HEADER_SIZE; using scaler::object_storage::CAPNP_WORD_SIZE; +class ServerClientTest: public ::testing::Test { +protected: + static void SetUpTestSuite() { + static std::once_flag server_started; + std::call_once(server_started, []() { + std::thread([] { run_object_storage_server("127.0.0.1", "55555"); }).detach(); + std::this_thread::sleep_for(std::chrono::seconds(1)); // Allow server to start + }); + } + + static void TearDownTestSuite() { + // No shutdown — server runs until process exits + } +}; + char payload[] = "Hello, world!"; awaitable async_read_response_header(tcp::socket& socket, scaler::object_storage::ObjectResponseHeader& header) { @@ -108,7 +126,7 @@ awaitable testSetObjectByID(tcp::socket socket) { printf("testSetObjectByID finished\n"); } -TEST(ObjectStorageTestSuite, TestHalt) { +TEST_F(ServerClientTest, TestHalt) { boost::asio::io_context io_context; tcp::socket socket(io_context); tcp::socket socket2(io_context); @@ -178,7 +196,7 @@ void write_payload(boost::asio::ip::tcp::socket& socket) { boost::asio::write(socket, buffer(payload)); } -TEST(ObjectStorageTestSuite, TestSetObject) { +TEST_F(ServerClientTest, TestSetObject) { boost::asio::io_context io_context; tcp::socket socket(io_context); tcp::resolver resolver(io_context); @@ -211,7 +229,7 @@ TEST(ObjectStorageTestSuite, TestSetObject) { (void)res; } -TEST(ObjectStorageTestSuite, TestGetObject) { +TEST_F(ServerClientTest, TestGetObject) { boost::asio::io_context io_context; tcp::socket socket(io_context); tcp::resolver resolver(io_context); @@ -272,7 +290,7 @@ TEST(ObjectStorageTestSuite, TestGetObject) { (void)res; } -TEST(ObjectStorageTestSuite, TestDeleteObject) { +TEST_F(ServerClientTest, TestDeleteObject) { boost::asio::io_context io_context; tcp::socket socket(io_context); tcp::resolver resolver(io_context); @@ -303,7 +321,7 @@ TEST(ObjectStorageTestSuite, TestDeleteObject) { (void)res; } -TEST(ObjectStorageTestSuite, TestEmptyObject) { +TEST_F(ServerClientTest, TestEmptyObject) { boost::asio::io_context io_context; tcp::socket socket(io_context); tcp::resolver resolver(io_context); diff --git a/scaler/object_storage/tests/test_update_record.cpp b/tests/object_storage/test_update_record.cpp similarity index 96% rename from scaler/object_storage/tests/test_update_record.cpp rename to tests/object_storage/test_update_record.cpp index bf74d7567..4088ddd46 100644 --- a/scaler/object_storage/tests/test_update_record.cpp +++ b/tests/object_storage/test_update_record.cpp @@ -1,11 +1,7 @@ - -#include -#include #include -#include "../defs.h" -#include "../object_storage_server.h" -#include "protocol/object_storage.capnp.h" +#include "scaler/object_storage/defs.h" +#include "scaler/object_storage/object_storage_server.h" using reqType = ObjectRequestHeader::ObjectRequestType; using respType = ObjectResponseHeader::ObjectResponseType;