feat: in-flight, streaming PartitionWriter
#10588
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| push: | |
| branches: [main, "rust-v*"] | |
| pull_request: | |
| branches: [main, "rust-v*", next, next/*] | |
| merge_group: | |
| env: | |
| SCCACHE_GHA_ENABLED: "true" | |
| RUSTC_WRAPPER: "sccache" | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest | |
| env: | |
| DEFAULT_FEATURES: "azure,datafusion,s3,gcs,glue,hdfs" | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| components: rustfmt,clippy | |
| cache: true | |
| # if we ever get rid of all the clippy warnings, we can remove this line | |
| rustflags: "" | |
| - name: Rustfmt Check | |
| uses: actions-rust-lang/rustfmt@v1 | |
| - name: build and lint with clippy | |
| run: cargo clippy --features ${{ env.DEFAULT_FEATURES }} --tests | |
| - name: Run Machete to check for unused dependencies | |
| uses: bnjbvr/cargo-machete@aa070c61641e482bc2b5d41c97b496ee4755bf37 | |
| test-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| targets: ${{ steps.matrix.outputs.os }} | |
| steps: | |
| - name: Define test matrix | |
| id: matrix | |
| run: | | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| echo 'os=["ubuntu-latest"]' >> "$GITHUB_OUTPUT" | |
| else | |
| echo 'os=["ubuntu-latest", "windows-latest", "macos-latest"]' >> "$GITHUB_OUTPUT" | |
| fi | |
| build: | |
| needs: test-matrix | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: ${{ fromJSON(needs.test-matrix.outputs.targets) }} | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| DEFAULT_FEATURES: "azure,datafusion,s3,gcs,glue,hdfs" | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| components: clippy | |
| cache: true | |
| # if we ever get rid of all the clippy warnings, we can remove this line | |
| rustflags: "" | |
| - name: Default build | |
| run: cargo build --package deltalake | |
| - name: Default features with clippy | |
| run: cargo clippy --features ${{ env.DEFAULT_FEATURES }} | |
| - name: Spot-check build for native-tls features | |
| run: cargo build --no-default-features --features azure,datafusion,s3-native-tls,gcs,glue | |
| - name: Check no default features (except rustls) | |
| run: cargo check --no-default-features --features rustls | |
| test: | |
| name: Unit Tests | |
| needs: test-matrix | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: ${{ fromJSON(needs.test-matrix.outputs.targets) }} | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| DEFAULT_FEATURES: "datafusion" | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| cache: true | |
| # if we ever get rid of all the clippy warnings, we can remove this line | |
| rustflags: "" | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Run tests | |
| shell: bash | |
| run: | | |
| make setup-dat | |
| cargo llvm-cov --features ${{ env.DEFAULT_FEATURES }} \ | |
| --workspace \ | |
| --codecov \ | |
| --output-path codecov.json \ | |
| --exclude deltalake \ | |
| --exclude deltalake-azure \ | |
| --exclude deltalake-aws \ | |
| --exclude deltalake-hdfs \ | |
| --exclude deltalake-lakefs | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: codecov.json | |
| fail_ci_if_error: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} |