merge upstream

Author: silezhou

.github/workflows/dev_pr/labeler.yml

@@ -29,11 +29,20 @@ sql:
 
 logical-expr:
 - changed-files:
-  - any-glob-to-any-file: ['datafusion/expr/**/*']
+  - any-glob-to-any-file: ['datafusion/expr/**/*', 'datafusion/expr-common/**/*']
+
+ffi:
+  - changed-files:
+      - any-glob-to-any-file: ['datafusion/ffi/**/*']
 
 physical-expr:
 - changed-files:
-  - any-glob-to-any-file: ['datafusion/physical-expr/**/*', 'datafusion/physical-expr-common/**/*', 'datafusion/physical-expr-aggregate/**/*', 'datafusion/physical-plan/**/*']
+  - any-glob-to-any-file: ['datafusion/physical-expr/**/*', 'datafusion/physical-expr-common/**/*', 'datafusion/physical-expr-aggregate/**/*']
+
+physical-plan:
+  - changed-files:
+      - any-glob-to-any-file: [datafusion/physical-plan/**/*']
+
 
 catalog:
   - changed-files:
@@ -47,6 +56,10 @@ execution:
   - changed-files:
       - any-glob-to-any-file: ['datafusion/execution/**/*']
 
+datasource:
+  - changed-files:
+      - any-glob-to-any-file: ['datafusion/datasource/**/*']
+
 functions:
   - changed-files:
       - any-glob-to-any-file: ['datafusion/functions/**/*', 'datafusion/functions-aggregate/**/*', 'datafusion/functions-aggregate-common', 'datafusion/functions-nested']

.github/workflows/docs.yaml

@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 on:
   push:
     branches:

.github/workflows/docs_pr.yaml

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# Tests for Docs that runs on PRs
 name: Docs
 
 concurrency:
@@ -48,7 +49,34 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
-      - name: Run doctests
+      - name: Run doctests (embedded rust examples)
         run: cargo test --doc --features avro,json
       - name: Verify Working Directory Clean
         run: git diff --exit-code
+
+  # Test doc build
+  linux-test-doc-build:
+    name: Test doc build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install doc dependencies
+        run: |
+          set -x
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install -r docs/requirements.txt
+      - name: Build docs html and check for warnings
+        run: |
+          set -x
+          source venv/bin/activate
+          cd docs
+          ./build.sh # fails on errors
+

.github/workflows/extended.yml

@@ -36,6 +36,7 @@ jobs:
   linux-build-lib:
     name: linux build test
     runs-on: ubuntu-latest
+    # note: do not use amd/rust container to preserve disk space
     steps:
       - uses: actions/checkout@v4
         with:
@@ -45,7 +46,7 @@ jobs:
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
           source $HOME/.cargo/env
-          rustup default stable
+          rustup toolchain install
       - name: Install Protobuf Compiler
         run: sudo apt-get install -y protobuf-compiler
       - name: Prepare cargo build
@@ -58,6 +59,7 @@ jobs:
     name: cargo test 'extended_tests' (amd64)
     needs: linux-build-lib
     runs-on: ubuntu-latest
+    # note: do not use amd/rust container to preserve disk space
     steps:
       - uses: actions/checkout@v4
         with:
@@ -69,7 +71,7 @@ jobs:
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
           source $HOME/.cargo/env
-          rustup default stable
+          rustup toolchain install
       - name: Install Protobuf Compiler
         run: sudo apt-get install -y protobuf-compiler
       # For debugging, test binaries can be large.
@@ -122,7 +124,7 @@ jobs:
           rust-version: stable
       - name: Run sqllogictest
         run: |
-          cargo test --profile release-nonlto --test sqllogictests -- --include-sqlite
+          cargo test --features backtrace --profile release-nonlto --test sqllogictests -- --include-sqlite
           cargo clean
 
 

.github/workflows/rust.yml

@@ -66,9 +66,12 @@ jobs:
           # the changes to `Cargo.lock` after building with the updated manifest.
           cargo check --profile ci --workspace --all-targets --features integration-tests --locked
 
-  # cargo check common, functions and substrait with no default features
-  linux-cargo-check-no-default-features:
-    name: cargo check no default features
+  # Check datafusion-common features
+  #
+  # Ensure via `cargo check` that the crate can be built with a
+  # subset of the features packages enabled.
+  linux-datafusion-common-features:
+    name: cargo check datafusion-common features
     needs: linux-build-lib
     runs-on: ubuntu-latest
     container:
@@ -79,28 +82,68 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
-      - name: Check datafusion without default features
-        # Some of the test binaries require the parquet feature still
-        #run: cargo check --all-targets --no-default-features -p datafusion
-        run: cargo check --profile ci --no-default-features -p datafusion
-
-      - name: Check datafusion-common without default features
+      - name: Check datafusion-common (no-default-features)
         run: cargo check --profile ci --all-targets --no-default-features -p datafusion-common
+        # Note: don't check other feature flags as datafusion-common is not typically used standalone
 
-      - name: Check datafusion-functions without default features
-        run: cargo check --profile ci --all-targets --no-default-features -p datafusion-functions
-
-      - name: Check datafusion-substrait without default features
+  # Check datafusion-substrait features
+  #
+  # Ensure via `cargo check` that the crate can be built with a
+  # subset of the features packages enabled.
+  linux-datafusion-substrait-features:
+    name: cargo check datafusion-substrait features
+    needs: linux-build-lib
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      - name: Check datafusion-substrait (no-default-features)
         run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait
+      - name: Check datafusion-substrait (physical)
+        run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=physical
+      - name: Install cmake
+        run: |
+          # note the builder setup runs apt-get update / installs protobuf compiler
+          apt-get install -y cmake
+      - name: Check datafusion-substrait (protoc)
+        run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=protoc
 
-      - name: Check workspace in debug mode
-        run: cargo check --profile ci --all-targets --workspace
-
-      - name: Check workspace with additional features
-        run: cargo check --profile ci --workspace --benches --features avro,json,integration-tests
-
-  # cargo check datafusion to ensure that the datafusion crate can be built with only a
-  # subset of the function packages enabled.
+  # Check datafusion-proto features
+  #
+  # Ensure via `cargo check` that the crate can be built with a
+  # subset of the features packages enabled.
+  linux-datafusion-proto-features:
+    name: cargo check datafusion-proto features
+    needs: linux-build-lib
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      - name: Check datafusion-proto (no-default-features)
+        run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto
+      # fails due to https://github.com/apache/datafusion/issues/15157
+      #- name: Check datafusion-proto (json)
+      #  run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=json
+      - name: Check datafusion-proto (parquet)
+        run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=parquet
+      - name: Check datafusion-proto (avro)
+        run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=avro
+
+
+  # Check datafusion crate features
+  #
+  # Ensure via `cargo check` that the crate can be built with a
+  # subset of the features packages enabled.
   linux-cargo-check-datafusion:
     name: cargo check datafusion
     needs: linux-build-lib
@@ -113,6 +156,11 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Check datafusion (no-default-features)
+        # Some of the test binaries require the parquet feature still
+        #run: cargo check --all-targets --no-default-features -p datafusion
+        run: cargo check --profile ci --no-default-features -p datafusion
+
       - name: Check datafusion (nested_expressions)
         run: cargo check --profile ci --no-default-features --features=nested_expressions -p datafusion
 
@@ -134,8 +182,10 @@ jobs:
       - name: Check datafusion (string_expressions)
         run: cargo check --profile ci --no-default-features --features=string_expressions -p datafusion
 
-  # cargo check datafusion-functions to ensure that the datafusion-functions crate can be built with
-  # only a subset of the function packages enabled.
+  # Check datafusion-functions crate features
+  #
+  # Ensure via `cargo check` that the crate can be built with a
+  # subset of the features packages enabled.
   linux-cargo-check-datafusion-functions:
     name: cargo check functions
     needs: linux-build-lib
@@ -148,6 +198,9 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Check datafusion-functions (no-default-features)
+        run: cargo check --profile ci --all-targets --no-default-features -p datafusion-functions
+
       - name: Check datafusion-functions (crypto)
         run: cargo check --profile ci --all-targets --no-default-features --features=crypto_expressions -p datafusion-functions
 
@@ -171,21 +224,41 @@ jobs:
     name: cargo test (amd64)
     needs: linux-build-lib
     runs-on: ubuntu-latest
-    container:
-      image: amd64/rust
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
       - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
+        run: rustup toolchain install stable
+      - name: Install Protobuf Compiler
+        run: sudo apt-get install -y protobuf-compiler
+      - name: Setup Minio - S3-compatible storage
+        run: |
+          docker run -d --name minio-container \
+            -p 9000:9000 \
+            -e MINIO_ROOT_USER=TEST-DataFusionLogin -e MINIO_ROOT_PASSWORD=TEST-DataFusionPassword \
+            -v $(pwd)/datafusion/core/tests/data:/source quay.io/minio/minio \
+            server /data
+          docker exec minio-container /bin/sh -c "\
+            mc ready local
+            mc alias set localminio http://localhost:9000 TEST-DataFusionLogin TEST-DataFusionPassword && \
+            mc mb localminio/data && \
+            mc cp -r /source/* localminio/data"
       - name: Run tests (excluding doctests)
+        env:
+          RUST_BACKTRACE: 1
+          AWS_ENDPOINT: http://127.0.0.1:9000
+          AWS_ACCESS_KEY_ID: TEST-DataFusionLogin
+          AWS_SECRET_ACCESS_KEY: TEST-DataFusionPassword
+          TEST_STORAGE_INTEGRATION: 1
+          AWS_ALLOW_HTTP: true
         run: cargo test --profile ci --exclude datafusion-examples --exclude ffi_example_table_provider --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests
       - name: Verify Working Directory Clean
         run: git diff --exit-code
+      - name: Minio Output
+        if: ${{ !cancelled() }}
+        run: docker logs minio-container
 
   linux-test-example:
     name: cargo examples (amd64)
@@ -259,6 +332,10 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: stable
+      - name: Install dependencies
+        run: |
+          apt-get update -qq
+          apt-get install -y -qq clang
       - name: Install wasm-pack
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - name: Build with wasm-pack
@@ -295,7 +372,7 @@ jobs:
           export RUST_MIN_STACK=20971520
           export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data`
           cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1
-          INCLUDE_TPCH=true cargo test --profile ci --package datafusion-sqllogictest --test sqllogictests
+          INCLUDE_TPCH=true cargo test --features backtrace --profile ci --package datafusion-sqllogictest --test sqllogictests
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
@@ -331,7 +408,7 @@ jobs:
       - name: Run sqllogictest
         run: |
           cd datafusion/sqllogictest
-          PG_COMPAT=true PG_URI="postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/db_test" cargo test --profile ci --features=postgres --test sqllogictests
+          PG_COMPAT=true PG_URI="postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/db_test" cargo test --features backtrace --profile ci --features=postgres --test sqllogictests
         env:
           # use postgres for the host here because we have specified a container for the job
           POSTGRES_HOST: postgres