ci: split out integration tests #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| push: | |
| branches: [main, "rust-v*"] | |
| pull_request: | |
| branches: [main, "rust-v*", next, next/*] | |
| merge_group: | |
| env: | |
| # https://github.com/rust-lang/cargo/issues/10280 | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| DEFAULT_FEATURES: "azure,datafusion,s3,gcs,glue,hdfs " | |
| # Disable full debug symbol generation to speed up CI build and keep memory down | |
| RUSTFLAGS: -C debuginfo=line-tables-only | |
| # Disable incremental builds by cargo for CI which should save disk space | |
| # and hopefully avoid final link "No space left on device" | |
| CARGO_INCREMENTAL: 0 | |
| SCCACHE_GHA_ENABLED: "true" | |
| RUSTC_WRAPPER: "sccache" | |
| jobs: | |
| cloud: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| target: | |
| - name: aws | |
| - name: azure | |
| - name: gcp | |
| env: | |
| AWS_DEFAULT_REGION: "us-east-1" | |
| AWS_ACCESS_KEY_ID: deltalake | |
| AWS_SECRET_ACCESS_KEY: weloverust | |
| AWS_ENDPOINT_URL: http://localhost:4566 | |
| AWS_ALLOW_HTTP: "1" | |
| AZURE_USE_EMULATOR: "1" | |
| AZURE_STORAGE_ALLOW_HTTP: "1" | |
| AZURITE_BLOB_STORAGE_URL: "http://localhost:10000" | |
| AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;" | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| cache: true | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Start emulated services | |
| run: docker compose up -d | |
| - name: Run tests with rustls (default) | |
| run: | | |
| gmake setup-dat | |
| cargo llvm-cov \ | |
| --package deltalake-${{ matrix.target.name }} \ | |
| --features integration_test \ | |
| --codecov \ | |
| --output-path codecov.json | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: codecov.json | |
| fail_ci_if_error: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| native_tls: | |
| name: aws-native-tls | |
| runs-on: ubuntu-latest | |
| env: | |
| AWS_DEFAULT_REGION: "us-east-1" | |
| AWS_ACCESS_KEY_ID: deltalake | |
| AWS_SECRET_ACCESS_KEY: weloverust | |
| AWS_ENDPOINT_URL: http://localhost:4566 | |
| AWS_ALLOW_HTTP: "1" | |
| AZURE_USE_EMULATOR: "1" | |
| AZURE_STORAGE_ALLOW_HTTP: "1" | |
| AZURITE_BLOB_STORAGE_URL: "http://localhost:10000" | |
| AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;" | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| cache: true | |
| - name: Start emulated services | |
| run: docker compose up -d | |
| - name: Run tests with native-tls | |
| run: | | |
| gmake setup-dat | |
| cargo test \ | |
| --package deltalake-aws | |
| --no-default-features \ | |
| --features integration_test,native-tls | |
| integration_test_hdfs: | |
| name: Integration Tests (HDFS) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| cache: true | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| # Install Java and Hadoop for HDFS integration tests | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: "temurin" | |
| java-version: "17" | |
| - name: Download Hadoop | |
| run: | | |
| wget -q https://dlcdn.apache.org/hadoop/common/hadoop-3.4.0/hadoop-3.4.0.tar.gz | |
| tar -xf hadoop-3.4.0.tar.gz -C $GITHUB_WORKSPACE | |
| echo "$GITHUB_WORKSPACE/hadoop-3.4.0/bin" >> $GITHUB_PATH | |
| - name: Run tests with rustls (default) | |
| run: | | |
| gmake setup-dat | |
| cargo llvm-cov \ | |
| --features integration_test \ | |
| --package deltalake-hdfs \ | |
| --codecov \ | |
| --output-path codecov.json | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: codecov.json | |
| fail_ci_if_error: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| integration_test_lakefs: | |
| name: Integration Tests (LakeFS v1.48) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Setup Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: stable | |
| cache: true | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Download Lakectl | |
| run: | | |
| wget -q https://github.com/treeverse/lakeFS/releases/download/v1.48.1/lakeFS_1.48.1_Linux_x86_64.tar.gz | |
| tar -xf lakeFS_1.48.1_Linux_x86_64.tar.gz -C $GITHUB_WORKSPACE | |
| echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH | |
| - name: Start emulated services | |
| run: docker compose -f docker-compose-lakefs.yml up -d | |
| - name: Run tests | |
| run: | | |
| gmake setup-dat | |
| cargo llvm-cov \ | |
| --package deltalake-lakefs \ | |
| --features integration_test_lakefs \ | |
| --codecov \ | |
| --output-path codecov.json | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: codecov.json | |
| fail_ci_if_error: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} |