enhance: add minio in cpp CI and split the build and test

.github/workflows/clang-format-ci.yml

@@ -0,0 +1,28 @@
+name: clang-format
+
+on:
+  push:
+   paths:
+    - 'cpp/**'
+    - '.github/workflows/clang-format-ci.yml'
+  pull_request:
+   paths:
+    - 'cpp/**'
+    - '.github/workflows/clang-format-ci.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: check-format
+        working-directory: ./cpp
+        run: 
+          make check-format
+
+

.github/workflows/cpp-ci.yml

@@ -4,52 +4,127 @@ on:
   push:
    paths:
     - 'cpp/**'
-    - '!go/**'
+    - '.github/workflows/cpp-ci.yml'
   pull_request:
    paths:
     - 'cpp/**'
-    - '!go/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  unittest:
+  build:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install dependencies
         uses: aminya/setup-cpp@v1
         with: 
           conan: 1.61.0
           cmake: true
 
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "cpp/src/format/bridge/rust -> target"
+
       - name: setup conan
         run: 
           conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
           && conan remote list
 
       - name: conan package cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.conan
-          key: conan-${{ hashFiles('./cpp/conanfile.py') }}
-          restore-keys: conan-
+          key: conan-cpp-${{ hashFiles('./cpp/conanfile.py') }}
+          restore-keys: conan-cpp-
 
       - name: Build
         working-directory: ./cpp
         run: |
-          make build USE_ASAN=True WITH_VORTEX=True BUILD_TYPE=Release
-
-      - name: check-format
+          make build USE_ASAN=True BUILD_TYPE=Release WITH_VORTEX=True
+
+      - name: Upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: gtest-binary
+          path: |
+            cpp/build/Release/test/milvus_test
+            cpp/build/Release/test/Test_FFI
+            cpp/build/Release/libs/
+            cpp/build/Release/libmilvus-storage.so*
+
+  lint:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install dependencies
+        uses: aminya/setup-cpp@v1
+        with:
+          conan: 1.61.0
+          cmake: true
+
+      - name: setup conan
+        run:
+          conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local --insert
+          && conan remote list
+
+      - name: conan package cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.conan
+          key: conan-cpp-${{ hashFiles('./cpp/conanfile.py') }}
+          restore-keys: conan-cpp-
+
+      - name: Build
         working-directory: ./cpp
-        run: 
-          make check-format
+        run: |
+          make build BUILD_TYPE=Release
 
       - name: check-tidy
         working-directory: ./cpp
-        run:
+        run: |
           make check-tidy
 
-      - name : Test
+  unittest:
+    runs-on: ubuntu-22.04
+    needs: build
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: gtest-binary
+          path: ./cpp/build/Release
+
+      - name: Change mod of binary
         working-directory: ./cpp
         run: |
-          cd build/Release/test && ./milvus_test && ./Test_FFI
+          chmod +x ./build/Release/test/milvus_test
+          chmod +x ./build/Release/test/Test_FFI
 
+      - name: Setup Minio
+        run: |
+          chmod +x ./cpp/scripts/setup_minio.sh
+          ./cpp/scripts/setup_minio.sh
+
+      - name: Run unittest with local filesystem
+        working-directory: ./cpp
+        run: |
+          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/build/Release:$(pwd)/build/Release/libs
+          ./build/Release/test/milvus_test && ./build/Release/test/Test_FFI
+
+      - name: Run unittest with minio
+        working-directory: ./cpp
+        run: |
+          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/build/Release:$(pwd)/build/Release/libs
+          source ./scripts/minio_env.sh && ./build/Release/test/milvus_test
+

cpp/Makefile

@@ -86,6 +86,14 @@ clean:
 test: build
 	cd build/Release/test && ./milvus_test && ./Test_FFI
 
+test-all: build
+	# running milvus_test and Test_FFI without minio
+	./build/Release/test/milvus_test && ./build/Release/test/Test_FFI
+
+	# running milvus_test and Test_FFI with minio
+	./scripts/setup_minio.sh
+	source ./scripts/minio_env.sh && ./build/Release/test/milvus_test
+
 # Test cloud storage with all providers or a specific one
 # Usage: make test-cloud-storage     # Test all cloud providers
 #        make test-cloud-storage aws  # Test AWS only

cpp/conanfile.py

@@ -221,6 +221,14 @@ def package(self):
         cmake = CMake(self)
         cmake.install()
         self.copy("*_c.h")
+
+    def imports(self):
+        dest_dir = "build/{}/libs".format(self.settings.build_type)
+
+        # export all dynamic libs
+        self.copy("*.dll", dst=dest_dir, src="bin")
+        self.copy("*.so*", dst=dest_dir, src="lib")
+        self.copy("*.dylib*", dst=dest_dir, src="lib")
 
     def package_info(self):
         self.cpp_info.set_property("cmake_file_name", "storage")

cpp/scripts/minio_env.sh

@@ -0,0 +1,7 @@
+export STORAGE_TYPE="remote"
+export CLOUD_PROVIDER="aws"
+export ADDRESS="http://localhost:9000"
+export BUCKET_NAME="test-bucket"
+export ACCESS_KEY="minioadmin"
+export SECRET_KEY="minioadmin"
+export REGION=""

cpp/scripts/setup_minio.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Check if Minio is already running locally (possibly outside Docker)
+if curl -s http://localhost:9000/minio/health/live &> /dev/null; then
+    echo "Minio is already running locally and healthy."
+else
+    # Check if Minio container exists in docker
+    if [ "$(docker ps -aq -f name=minio)" ]; then
+        if [ ! "$(docker ps -q -f name=minio)" ]; then
+            echo "Starting existing minio container..."
+            docker start minio
+        else
+            echo "Minio container is already running."
+        fi
+    else
+        echo "Creating and starting new minio container..."
+        docker run -d -p 9000:9000 -p 9001:9001 --name minio \
+          -e "MINIO_ACCESS_KEY=minioadmin" \
+          -e "MINIO_SECRET_KEY=minioadmin" \
+          quay.io/minio/minio server /data --console-address ":9001"
+    fi
+fi
+
+# Install s3cmd if not present
+if ! command -v s3cmd &> /dev/null; then
+    echo "Installing s3cmd..."
+    if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+        sudo apt-get update && sudo apt-get install -y s3cmd
+    elif [[ "$OSTYPE" == "darwin"* ]]; then
+        if command -v brew &> /dev/null; then
+            brew install s3cmd
+        else
+            echo "Homebrew not found. Please install s3cmd manually."
+            exit 1
+        fi
+    else
+        echo "Unsupported OS for automatic s3cmd installation. Please install s3cmd manually."
+        exit 1
+    fi
+fi
+
+# Configure s3cmd for minio
+cat <<EOF > ~/.s3cfg
+[default]
+access_key = minioadmin
+secret_key = minioadmin
+host_base = localhost:9000
+host_bucket = localhost:9000
+use_https = False
+EOF
+
+# Wait for minio to be ready
+echo "Waiting for Minio to be ready..."
+max_retries=10
+count=0
+while [ $count -lt $max_retries ]; do
+  if curl -s http://localhost:9000/minio/health/live; then
+    echo "Minio is ready"
+    break
+  fi
+  echo "Still waiting for Minio... ($((count + 1))/$max_retries)"
+  sleep 2
+  count=$((count + 1))
+done
+
+if [ $count -eq $max_retries ]; then
+    echo "Minio failed to start in time"
+    exit 1
+fi
+
+# Create bucket for test if not exists
+if s3cmd ls s3://test-bucket &> /dev/null; then
+    echo "Bucket test-bucket already exists"
+else
+    echo "Creating bucket test-bucket..."
+    s3cmd mb s3://test-bucket
+fi

cpp/src/properties.cpp

@@ -383,7 +383,7 @@ static std::unordered_map<std::string, PropertyInfo> property_infos = {
     REGISTER_PROPERTY(PROPERTY_FS_LOG_LEVEL,
                       PropertyType::STRING,
                       "The log level for the filesystem storage service.",
-                      "warn",
+                      "fatal",
                       ValidatePropertyType() +
                           ValidatePropertyEnum<std::string>("fatal", "error", "warn", "info", "debug", "trace", "off")),
     REGISTER_PROPERTY(

cpp/test/CMakeLists.txt

@@ -22,6 +22,7 @@ set_target_properties(milvus_test PROPERTIES
   CXX_STANDARD 17
 )
 
+
 target_link_libraries(
   milvus_test PRIVATE milvus-storage GTest::gtest_main
 )

cpp/test/s3_file_system_test.cpp

@@ -75,7 +75,9 @@ TEST_F(S3FsTest, ConditionalWrite) {
 
     auto conditional_fs = std::dynamic_pointer_cast<UploadConditional>(fs_);
     ASSERT_NE(conditional_fs, nullptr);
-    ASSERT_STATUS_NOT_OK(conditional_fs->OpenConditionalOutputStream(file_to, nullptr));
+    ASSERT_AND_ASSIGN(auto output_stream, conditional_fs->OpenConditionalOutputStream(file_to, nullptr));
+    ASSERT_STATUS_OK(output_stream->Write(buffer));
+    ASSERT_STATUS_NOT_OK(output_stream->Close());
   }
 
   (void)fs_->DeleteFile(file_to);