chore(ci): migrate Linux CI to runs-on self-hosted runners #7792
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This is mostly copied from the rust-analyzer repo | |
| # https://github.com/rust-lang/rust-analyzer/blob/12e7aa3132217cc6a6c1151d468be35d7b365999/.github/workflows/ci.yaml | |
| name: Rust CI | |
| on: | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled] | |
| merge_group: | |
| push: | |
| branches: | |
| - main | |
| - 'support/**' | |
| schedule: | |
| # three times a day to run the integration tests that take a long time | |
| - cron: '33 3,10,15 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_INCREMENTAL: 0 | |
| CARGO_NET_RETRY: 10 | |
| CI: 1 | |
| RUST_BACKTRACE: short | |
| RUSTFLAGS: "-D warnings" | |
| RUSTUP_MAX_RETRIES: 10 | |
| RUST_CHANNEL: '1.91.1' | |
| CARGO_DENY_VERSION: '0.19.0' | |
| jobs: | |
| rust: | |
| name: Rust CI | |
| timeout-minutes: 20 | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| working-directory: ./ | |
| #permissions: | |
| #contents: read | |
| #actions: read | |
| #pull-requests: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ["runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/extras=s3-cache", "runs-on=${{ github.run_id }}/runner=4cpu-linux-arm64/extras=s3-cache", macos-15-intel, macos-latest] | |
| steps: | |
| - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2 | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Check if MSRV is up to date | |
| run: | | |
| set -e | |
| grep 'rust-version = "${{ env.RUST_CHANNEL }}"' Cargo.toml | |
| # shellcheck disable=SC2016 | |
| grep 'The current MSRV is `${{ env.RUST_CHANNEL }}`' icechunk-python/docs/docs/reference/contributing.md | |
| - name: Stand up docker services | |
| if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu') | |
| run: | | |
| docker compose up -d | |
| - name: Wait for containers to be ready | |
| if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu') | |
| run: | | |
| for _ in {1..30}; do | |
| if docker compose ps --status exited --filter status==0 | grep rustfs ; then | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| for _ in {1..60}; do | |
| if curl --silent --fail "http://localhost:10000/devstoreaccount1/testcontainer?sv=2023-01-03&ss=btqf&srt=sco&spr=https%2Chttp&st=2025-01-06T14%3A53%3A30Z&se=2035-01-07T14%3A53%3A00Z&sp=rwdftlacup&sig=jclETGilOzONYp4Y0iK9SpVRLGyehaS5lg5booJ9VYA%3D&restype=container"; then | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| - name: Install Just | |
| if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu') | |
| uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # v2 | |
| with: | |
| tool: just | |
| - name: Install cargo nextest | |
| uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # nextest | |
| - name: Install Rust toolchain | |
| run: | | |
| rustup update --no-self-update ${{ env.RUST_CHANNEL }} | |
| rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy | |
| rustup default ${{ env.RUST_CHANNEL }} | |
| - name: Cache Dependencies | |
| uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 | |
| with: | |
| # workspaces: "rust -> target" | |
| key: ${{ env.RUST_CHANNEL }} | |
| - name: Run tests with Docker services | |
| if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu') | |
| env: | |
| RUST_LOG: icechunk=trace | |
| R2_BUCKET: ${{ secrets.R2_BUCKET }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} | |
| AWS_BUCKET: ${{ secrets.AWS_BUCKET }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| TIGRIS_BUCKET: ${{ secrets.TIGRIS_BUCKET }} | |
| TIGRIS_REGION: ${{ secrets.TIGRIS_REGION }} | |
| TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} | |
| TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} | |
| run: | | |
| just profile=ci compile-tests "--locked" | |
| just profile=ci test | |
| - name: Run doc tests | |
| if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu') | |
| run: | | |
| just profile=ci doctest | |
| - name: Run examples | |
| if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu') | |
| run: | | |
| just profile=ci run-all-examples | |
| - name: Run unit tests only | |
| if: ${{ !contains(matrix.os, 'linux') && !contains(matrix.os, 'ubuntu') }} | |
| run: | | |
| cargo test --lib --profile ci | |
| - name: Run integration tests against object stores | |
| if: github.event_name == 'cron' | |
| env: | |
| R2_BUCKET: ${{ secrets.R2_BUCKET }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} | |
| AWS_BUCKET: ${{ secrets.AWS_BUCKET }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| TIGRIS_BUCKET: ${{ secrets.TIGRIS_BUCKET }} | |
| TIGRIS_REGION: ${{ secrets.TIGRIS_REGION }} | |
| TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }} | |
| TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }} | |
| run: | | |
| cargo test --profile ci --workspace --all-targets -- --ignored | |
| # shuttle: | |
| # name: Shuttle concurrency tests | |
| # timeout-minutes: 30 | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - name: Checkout repository | |
| # uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| # with: | |
| # ref: ${{ github.event.pull_request.head.sha }} | |
| # - name: Install Rust toolchain | |
| # run: | | |
| # rustup update --no-self-update ${{ env.RUST_CHANNEL }} | |
| # rustup default ${{ env.RUST_CHANNEL }} | |
| # - name: Cache Dependencies | |
| # uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 | |
| # with: | |
| # key: shuttle-${{ env.RUST_CHANNEL }} | |
| # - name: Compile shuttle tests | |
| # run: cargo test -p icechunk --features shuttle --test test_shuttle --no-run | |
| # - name: Run shuttle tests | |
| # run: cargo test -p icechunk --features shuttle --test test_shuttle -- --nocapture | |
| wasm-build: | |
| name: WASM Build | |
| timeout-minutes: 15 | |
| runs-on: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/extras=s3-cache | |
| defaults: | |
| run: | |
| working-directory: ./ | |
| env: | |
| # Keep this as a compile smoke test; avoid failing on existing warnings in | |
| # no-default-features wasm cfg combinations. | |
| RUSTFLAGS: "" | |
| steps: | |
| - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2 | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Install Rust toolchain | |
| run: | | |
| rustup update --no-self-update ${{ env.RUST_CHANNEL }} | |
| rustup default ${{ env.RUST_CHANNEL }} | |
| rustup target add wasm32-wasip1-threads | |
| - name: Install LLVM toolchain for wasm C deps | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends clang llvm wasi-libc libc++-dev | |
| - name: Show wasm clang include search paths | |
| run: | | |
| echo | clang --target=wasm32-wasi -E -x c - -v | |
| echo '#include <limits.h>' | clang --target=wasm32-wasi -E -x c - -v > /dev/null | |
| ls -la /usr/include/wasm32-wasi || true | |
| ls -la /usr/lib/wasm32-wasi || true | |
| test -f /usr/include/wasm32-wasi/limits.h && echo "found /usr/include/wasm32-wasi/limits.h" | |
| - name: Cache Dependencies | |
| uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 | |
| with: | |
| key: wasm-${{ env.RUST_CHANNEL }} | |
| - name: Run wasm build check | |
| env: | |
| CC_wasm32_wasip1_threads: clang | |
| CXX_wasm32_wasip1_threads: clang++ | |
| AR_wasm32_wasip1_threads: llvm-ar | |
| CC_wasm32_wasi: clang | |
| CXX_wasm32_wasi: clang++ | |
| AR_wasm32_wasi: llvm-ar | |
| WASI_SYSROOT: /usr | |
| CFLAGS_wasm32_wasip1_threads: --sysroot=/usr -isystem /usr/include/wasm32-wasi | |
| CXXFLAGS_wasm32_wasip1_threads: --sysroot=/usr -isystem /usr/include/wasm32-wasi | |
| CFLAGS_wasm32_wasi: --sysroot=/usr -isystem /usr/include/wasm32-wasi | |
| CXXFLAGS_wasm32_wasi: --sysroot=/usr -isystem /usr/include/wasm32-wasi | |
| run: | | |
| cargo build -p icechunk --no-default-features --target wasm32-wasip1-threads | |
| - name: Run no-default-features lib tests (proxy for wasm feature set) | |
| run: | | |
| cargo test -p icechunk --no-default-features --lib |