Merge branch 'main' into nmh/add-undelete-rfc

Author: NikitaMatskevich

.asf.yaml

@@ -32,6 +32,10 @@ github:
     squash: true
     merge: false
     rebase: false
+  pull_requests:
+    allow_auto_merge: true
+    allow_update_branch: true
+    del_branch_on_merge: true
   protected_branches:
     main:
       required_pull_request_reviews:

.devcontainer/post_create.sh

@@ -72,3 +72,9 @@ sudo apt install -y ninja-build
 
 # Setup for D binding
 sudo apt install -y dmd dub
+
+# Setup just for development tasks
+cargo install just
+
+# Setup uv for python project management
+curl -LsSf https://astral.sh/uv/install.sh | sh

.env.example

@@ -88,12 +88,6 @@ OPENDAL_WEBHDFS_DISABLE_LIST_BATCH=false
 OPENDAL_VERCEL_ARTIFACTS_ACCESS_TOKEN=<token>
 # onedrive
 OPENDAL_ONEDRIVE_ACCESS_TOKEN=<access_token>
-# atomicserver
-OPENDAL_ATOMICSERVER_ROOT=/path/to/dir
-OPENDAL_ATOMICSERVER_ENDPOINT=http://localhost:9883
-OPENDAL_ATOMICSERVER_PRIVATE_KEY=<private_key>
-OPENDAL_ATOMICSERVER_PUBLIC_KEY=<public_key>
-OPENDAL_ATOMICSERVER_PARENT_RESOURCE_ID=http://localhost:9883
 # foundationdb
 OPENDAL_FOUNDATIONDB_ROOT=/path/to/dir
 OPENDAL_FOUNDATIONDB_CONFIG_PATH=/tmp/opendal/foundationdb.conf
@@ -192,13 +186,6 @@ OPENDAL_KOOFR_ROOT=/path/to/dir
 OPENDAL_KOOFR_ENDPOINT=<endpoint>
 OPENDAL_KOOFR_EMAIL=<email>
 OPENDAL_KOOFR_PASSWORD=<password>
-# icloud
-OPENDAL_ICLOUD_ROOT=/
-OPENDAL_ICLOUD_APPLE_ID=<apple_id>
-OPENDAL_ICLOUD_PASSWORD=<password>
-OPENDAL_ICLOUD_TRUST_TOKEN=<trust_token>
-OPENDAL_ICLOUD_DS_WEB_AUTH_TOKEN=<ds_web_auth_token>
-OPENDAL_ICLOUD_IS_CHINA_MAINLAND=true
 # vercel blob
 OPENDAL_VERCEL_BLOB_ROOT=/path/to/dir
 OPENDAL_VERCEL_BLOB_TOKEN=<token>
@@ -207,3 +194,8 @@ OPENDAL_ALIYUN_DRIVE_ROOT=/path/to/dir
 OPENDAL_ALIYUN_DRIVE_REFRESH_TOKEN=<refresh_token>
 OPENDAL_ALIYUN_DRIVE_CLIENT_ID=<client_id>
 OPENDAL_ALIYUN_DRIVE_CLIENT_SECRET=<client_secret>
+# cloudflare
+OPENDAL_CLOUDFLARE_KV_ROOT=/path/to/dir
+OPENDAL_CLOUDFLARE_KV_API_TOKEN=<api_token>
+OPENDAL_CLOUDFLARE_KV_ACCOUNT_ID=<account_id>
+OPENDAL_CLOUDFLARE_KV_NAMESPACE_ID=<namespace_id>

.github/CODEOWNERS

@@ -1,4 +1,4 @@
-* @Xuanwo
+* @Xuanwo @tisonkun
 
 /core @Xuanwo
 /bindings/c/ @xyjixyjixyji

.github/actions/fuzz_test/action.yaml

@@ -24,6 +24,8 @@ inputs:
     description: "The service to test"
   target:
     description: "The fuzz target to test"
+  toolchain:
+    description: "The rust toolchain to use"
 
 runs:
   using: "composite"
@@ -41,7 +43,7 @@ runs:
           - name: Run Fuzz Test
             shell: bash
             working-directory: core/fuzz
-            run: cargo +nightly-2024-11-22 fuzz run ${{ inputs.target }} --features services-${{ inputs.service }} -- -max_total_time=120
+            run: cargo +${{ inputs.toolchain }} fuzz run ${{ inputs.target }} --features services-${{ inputs.service }} -- -max_total_time=120
             env:
               OPENDAL_TEST: ${{ inputs.service }}
         EOF

.github/actions/setup-ocaml/action.yaml

@@ -32,6 +32,7 @@ runs:
         ocaml-compiler: 4.14
         opam-depext: ${{inputs.need-depext == true}}
         opam-pin: ${{inputs.need-pin == true}}
+        dune-cache: true
 
     - name: Set Opam env
       shell: bash

.github/actions/setup/action.yaml

@@ -16,7 +16,7 @@
 # under the License.
 
 name: Setup Rust Builder
-description: 'Prepare Rust Build Environment'
+description: "Prepare Rust Build Environment"
 inputs:
   need-rocksdb:
     description: "This setup needs rocksdb or not"
@@ -59,10 +59,83 @@ runs:
 
     - name: Setup Protoc
       if: inputs.need-protoc == 'true'
-      uses: arduino/setup-protoc@v3
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        PROTOC_VERSION="23.4"
+        PROTOC_ROOT="${RUNNER_TEMP}/protoc/${PROTOC_VERSION}"
+
+        case "${RUNNER_OS}-${RUNNER_ARCH}" in
+          Linux-X64)
+            PROTOC_PLATFORM="linux-x86_64"
+            ;;
+          Linux-ARM64)
+            PROTOC_PLATFORM="linux-aarch_64"
+            ;;
+          macOS-X64)
+            PROTOC_PLATFORM="osx-x86_64"
+            ;;
+          macOS-ARM64)
+            PROTOC_PLATFORM="osx-aarch_64"
+            ;;
+          Windows-X64)
+            PROTOC_PLATFORM="win64"
+            ;;
+          *)
+            echo "Unsupported runner: ${RUNNER_OS}-${RUNNER_ARCH}" >&2
+            exit 1
+            ;;
+        esac
+
+        echo "PROTOC_VERSION=${PROTOC_VERSION}" >> "${GITHUB_ENV}"
+        echo "PROTOC_ROOT=${PROTOC_ROOT}" >> "${GITHUB_ENV}"
+        echo "PROTOC_PLATFORM=${PROTOC_PLATFORM}" >> "${GITHUB_ENV}"
+
+    - name: Cache Protoc
+      if: inputs.need-protoc == 'true'
+      id: cache-protoc
+      uses: actions/cache@v4
       with:
-        version: "23.4"
-        repo-token: ${{ inputs.github-token }}
+        path: ${{ runner.temp }}/protoc/${{ env.PROTOC_VERSION }}
+        key: protoc-${{ env.PROTOC_VERSION }}-${{ runner.os }}-${{ runner.arch }}
+
+    - name: Install Protoc (Unix)
+      if: inputs.need-protoc == 'true' && runner.os != 'Windows' && steps.cache-protoc.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        PROTOC_ZIP="protoc-${PROTOC_VERSION}-${PROTOC_PLATFORM}.zip"
+        PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/${PROTOC_ZIP}"
+
+        mkdir -p "${PROTOC_ROOT}"
+        curl -fsSL -o "${RUNNER_TEMP}/${PROTOC_ZIP}" "${PROTOC_URL}"
+        unzip -q "${RUNNER_TEMP}/${PROTOC_ZIP}" -d "${PROTOC_ROOT}"
+
+    - name: Install Protoc (Windows)
+      if: inputs.need-protoc == 'true' && runner.os == 'Windows' && steps.cache-protoc.outputs.cache-hit != 'true'
+      shell: pwsh
+      run: |
+        $ErrorActionPreference = 'Stop'
+
+        $protocZip = "protoc-$env:PROTOC_VERSION-$env:PROTOC_PLATFORM.zip"
+        $protocUrl = "https://github.com/protocolbuffers/protobuf/releases/download/v$env:PROTOC_VERSION/$protocZip"
+        $zipPath = Join-Path $env:RUNNER_TEMP $protocZip
+        $dest = $env:PROTOC_ROOT
+
+        New-Item -ItemType Directory -Force -Path $dest | Out-Null
+        Invoke-WebRequest -Uri $protocUrl -OutFile $zipPath
+        Expand-Archive -Path $zipPath -DestinationPath $dest -Force
+
+    - name: Add Protoc to PATH
+      if: inputs.need-protoc == 'true'
+      shell: bash
+      run: |
+        set -euo pipefail
+        echo "${PROTOC_ROOT}/bin" >> "${GITHUB_PATH}"
+        export PATH="${PROTOC_ROOT}/bin:${PATH}"
+        protoc --version
 
     - name: Install cargo-nextest
       if: inputs.need-nextest == 'true'
@@ -133,4 +206,3 @@ runs:
 
         rm foundationdb-clients_7.1.17-1_amd64.deb
         rm foundationdb-server_7.1.17-1_amd64.deb
-

.github/actions/test_behavior_bin_ofs/action.yaml

@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Test Binding C++
+description: 'Test Core with given setup and service'
+inputs:
+  setup:
+    description: "The setup action for test"
+  service:
+    description: "The service to test"
+  feature:
+    description: "The feature to test"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup
+      shell: bash
+      run: |
+        mkdir -p ./dynamic_test_binding_cpp &&
+        cat <<EOF >./dynamic_test_binding_cpp/action.yml
+        runs:
+          using: composite
+          steps:
+          - name: Setup Test
+            uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
+          - name: Run Test Binding C++
+            shell: bash
+            working-directory: bindings/cpp
+            run: |
+              mkdir -p build && cd build
+              if [[ "\${{ runner.os }}" == "Windows" ]]; then
+                cmake \
+                  -DCMAKE_BUILD_TYPE=Release \
+                  -DOPENDAL_ENABLE_TESTING=ON \
+                  -DOPENDAL_DEV=ON \
+                  -DOPENDAL_ENABLE_ASYNC=ON \
+                  -DFEATURES="opendal/${{ inputs.feature }}" \
+                  -DCMAKE_CXX_STANDARD=20 \
+                  ..
+                cmake --build . --config Release
+                ./Release/opendal_cpp_test.exe --gtest_filter="*BehaviorTest*:*Async*"
+              elif [[ "\${{ runner.os }}" == "macOS" ]]; then
+                cmake \
+                  -DCMAKE_BUILD_TYPE=Release \
+                  -DOPENDAL_ENABLE_TESTING=ON \
+                  -DOPENDAL_DEV=ON \
+                  -DOPENDAL_ENABLE_ASYNC=ON \
+                  -DFEATURES="opendal/${{ inputs.feature }}" \
+                  -DCMAKE_CXX_COMPILER=clang++ \
+                  -DCMAKE_CXX_STANDARD=20 \
+                  ..
+                make -j\$(sysctl -n hw.ncpu) opendal_cpp_test
+                ./opendal_cpp_test --gtest_filter="*BehaviorTest*:*Async*"
+              else
+                cmake \
+                  -DCMAKE_BUILD_TYPE=Release \
+                  -DOPENDAL_ENABLE_TESTING=ON \
+                  -DOPENDAL_DEV=ON \
+                  -DOPENDAL_ENABLE_ASYNC=ON \
+                  -DFEATURES="opendal/${{ inputs.feature }}" \
+                  -DCMAKE_CXX_COMPILER=clang++ \
+                  -DCMAKE_CXX_STANDARD=20 \
+                  ..
+                make -j\$(nproc) opendal_cpp_test
+                ./opendal_cpp_test --gtest_filter="*BehaviorTest*:*Async*"
+              fi
+            env:
+              OPENDAL_TEST: ${{ inputs.service }}
+        EOF
+    - name: Run
+      uses: ./dynamic_test_binding_cpp 

.github/actions/test_behavior_binding_cpp/action.yaml

@@ -40,10 +40,16 @@ runs:
             uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
           - name: Run Test Binding Go
             shell: bash
-            working-directory: bindings/go/tests/behavior_tests
             run: |
-              export DYLD_FALLBACK_LIBRARY_PATH=$DYLD_FALLBACK_LIBRARY_PATH:/opt/homebrew/opt/libffi/lib
-              go test -v -run TestBehavior
+              if [ "${{ runner.os }}" == "Windows" ]; then
+                # https://github.com/ilammy/msvc-dev-cmd/issues/25
+                rm /usr/bin/link.exe
+              fi
+              cp bindings/go/Makefile .
+              make tests
+              if [ "${{ inputs.setup }}" == "0_minio_s3" ]; then
+                make bench
+              fi
             env:
               OPENDAL_TEST: ${{ inputs.service }}
               CGO_ENABLE: 0