[pull] trunk from spiceai:trunk #3463
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: pr | |
| on: | |
| pull_request: | |
| branches: | |
| - trunk | |
| - release-* | |
| - release/* | |
| - feature-* | |
| merge_group: | |
| branches: | |
| - trunk | |
| - release-* | |
| - release/* | |
| - feature-* | |
| workflow_dispatch: | |
| inputs: | |
| profile_option: | |
| description: 'Which Rust build profile to use? (Default is lint)' | |
| required: false | |
| type: choice | |
| options: | |
| - 'lint' | |
| - 'release' | |
| - 'release-lto' | |
| default: 'lint' | |
| update_snapshots: | |
| description: 'Update the snapshots?' | |
| required: false | |
| default: 'no' | |
| type: choice | |
| options: | |
| - 'always' | |
| - 'no' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'trunk' && github.sha || 'any-sha' }} | |
| cancel-in-progress: true | |
| env: | |
| IGNORED_PATHS: '(^./docs/|^README.md$|^version.txt$|^acknowledgements.md$)' | |
| RUST_PROFILE: ${{ github.event.inputs.profile_option || 'lint' }} | |
| # CI performance optimizations | |
| CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse | |
| CARGO_NET_RETRY: 10 | |
| CARGO_HTTP_TIMEOUT: 60 | |
| CARGO_INCREMENTAL: 0 | |
| CARGO_NET_GIT_FETCH_WITH_CLI: true | |
| # Use fast linkers for PR builds (mold for x86_64, lld for aarch64) | |
| CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: clang | |
| CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -C link-arg=-fuse-ld=mold | |
| CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: clang | |
| CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -C link-arg=-fuse-ld=lld | |
| jobs: | |
| # `pr` is a required check for pull requests. Therefore we cannot skip the workflow via | |
| # `on.pull_requests.paths-ignore`. Instead, we conditionally run the jobs (which report | |
| # a success when skipped). | |
| check_changes: | |
| runs-on: spiceai-dev-runners | |
| outputs: | |
| relevant_changes: ${{ steps.check.outputs.relevant_changes }} | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| with: | |
| # Use fetch-depth 2 for PR change detection (dorny/paths-filter only needs base + head) | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} | |
| - name: Check for code changes | |
| id: check | |
| uses: ./.github/actions/check-code-changes | |
| - name: Validate datafusion-table-providers commit | |
| run: bash .github/scripts/validate_table_providers_commit.sh | |
| env: | |
| BRANCH: spiceai-52 | |
| # Run Rust linting | |
| lint-rust: | |
| name: Rust Lint | |
| runs-on: spiceai-dev-runners | |
| needs: check_changes | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| with: | |
| fetch-depth: 1 | |
| - name: Cache Cargo registry | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 | |
| with: | |
| path: | | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Set up Rust | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-rust | |
| - name: Set up sccache | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-sccache | |
| with: | |
| minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }} | |
| - name: Install Protoc | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up make | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-make | |
| - name: Set up cc | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-cc | |
| - name: Run Rust lint | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| run: make lint-rust | |
| env: | |
| RUSTC_WRAPPER: ${{ env.RUSTC_WRAPPER }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }} | |
| RUST_PROFILE: ${{ env.RUST_PROFILE }} | |
| - name: Cancel workflow on failure | |
| if: failure() | |
| run: gh run cancel ${{ github.run_id }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Show sccache stats | |
| if: ${{ needs.check_changes.outputs.relevant_changes == 'true' && env.SCCACHE_SETUP == 'true' }} | |
| run: sccache --show-stats | |
| # Build and test | |
| build-test: | |
| name: Build and Test | |
| runs-on: spiceai-dev-runners | |
| needs: [check_changes] | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| with: | |
| fetch-depth: 1 | |
| - name: Cache Cargo registry | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 | |
| with: | |
| path: | | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Set up Rust | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-rust | |
| - name: Set up Nextest | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-nextest | |
| - name: Check if sccache can be set up | |
| run: | | |
| if [ -z "${{ secrets.TEST_MINIO_ENDPOINT }}" ]; then | |
| echo "SCCACHE_SETUP=false" >> $GITHUB_ENV | |
| echo "RUSTC_WRAPPER=" >> $GITHUB_ENV | |
| else | |
| echo "SCCACHE_SETUP=true" >> $GITHUB_ENV | |
| echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV | |
| fi | |
| - name: Set up sccache | |
| if: ${{ needs.check_changes.outputs.relevant_changes == 'true' && env.SCCACHE_SETUP == 'true' }} | |
| uses: ./.github/actions/setup-sccache | |
| with: | |
| minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }} | |
| - name: Install Protoc | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up make | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-make | |
| - name: Set up cc | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| uses: ./.github/actions/setup-cc | |
| - name: Build CLI and run nextest | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| run: make build-cli nextest | |
| env: | |
| RUSTC_WRAPPER: ${{ env.RUSTC_WRAPPER }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }} | |
| RUST_PROFILE: ${{ env.RUST_PROFILE }} | |
| AWS_S3_CLIENT_ID: ${{ secrets.AWS_S3_CLIENT_ID }} | |
| AWS_S3_IDENTITY_POOL_ID: ${{ secrets.AWS_S3_IDENTITY_POOL_ID }} | |
| AWS_S3_USERNAME: ${{ secrets.AWS_S3_USERNAME }} | |
| AWS_S3_PASSWORD: ${{ secrets.AWS_S3_PASSWORD }} | |
| AWS_COGNITO_IDP_URI: ${{ vars.AWS_COGNITO_IDP_URI }} | |
| AWS_REGION: ${{ vars.AWS_REGION }} | |
| AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }} | |
| INSTA_UPDATE: ${{ github.event.inputs.update_snapshots || 'no' }} | |
| - name: Cancel workflow on failure | |
| if: failure() | |
| run: gh run cancel ${{ github.run_id }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Build testoperator | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| env: | |
| RUSTC_WRAPPER: ${{ env.RUSTC_WRAPPER }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.TEST_MINIO_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_MINIO_SECRET_KEY }} | |
| run: cargo build -p testoperator --all-features --locked | |
| - name: Show sccache stats | |
| if: ${{ needs.check_changes.outputs.relevant_changes == 'true' && env.SCCACHE_SETUP == 'true' }} | |
| run: sccache --show-stats | |
| - name: Check if Cargo.lock is updated | |
| if: needs.check_changes.outputs.relevant_changes == 'true' | |
| run: | | |
| if git diff --exit-code Cargo.lock; then | |
| echo "Cargo.lock is up to date" | |
| else | |
| echo "Update Cargo.lock" | |
| exit 1 | |
| fi | |
| - name: Push snapshots to branch | |
| if: github.event.inputs.update_snapshots == 'always' | |
| uses: ./.github/actions/push-snap-changes | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} |