rust-ci.yaml

# This is mostly copied from the rust-analyzer repo
# https://github.com/rust-lang/rust-analyzer/blob/12e7aa3132217cc6a6c1151d468be35d7b365999/.github/workflows/ci.yaml

name: Rust CI
on:
  pull_request:
    types: [opened, reopened, synchronize, labeled]
  merge_group:
  push:
    branches:
      - main
      - 'support/**'
  schedule:
    # three times a day to run the integration tests that take a long time
    - cron:  '33 3,10,15 * * *'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  CI: 1
  RUST_BACKTRACE: short
  RUSTFLAGS: "-D warnings"
  RUSTUP_MAX_RETRIES: 10
  RUST_CHANNEL: '1.91.1'
  CARGO_DENY_VERSION: '0.19.0'

jobs:
  rust:
    name: Rust CI
    timeout-minutes: 20
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        working-directory: ./
    #permissions:
      #contents: read
      #actions: read
      #pull-requests: read

    strategy:
      fail-fast: false
      matrix:
        os: ["runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/extras=s3-cache", "runs-on=${{ github.run_id }}/runner=4cpu-linux-arm64/extras=s3-cache", macos-15-intel, macos-latest]

    steps:
      - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Check if MSRV is up to date
        run: |
          set -e
          grep 'rust-version = "${{ env.RUST_CHANNEL }}"' Cargo.toml
          # shellcheck disable=SC2016
          grep 'The current MSRV is `${{ env.RUST_CHANNEL }}`' icechunk-python/docs/docs/reference/contributing.md

      - name: Stand up docker services
        if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu')
        run: |
          docker compose up -d

      - name: Wait for containers to be ready
        if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu')
        run: |
          for _ in {1..30}; do
              if docker compose ps --status exited --filter status==0 | grep rustfs ; then
              break
              fi
              sleep 1
          done
          for _ in {1..60}; do
              if curl --silent --fail "http://localhost:10000/devstoreaccount1/testcontainer?sv=2023-01-03&ss=btqf&srt=sco&spr=https%2Chttp&st=2025-01-06T14%3A53%3A30Z&se=2035-01-07T14%3A53%3A00Z&sp=rwdftlacup&sig=jclETGilOzONYp4Y0iK9SpVRLGyehaS5lg5booJ9VYA%3D&restype=container"; then
              break
              fi
              sleep 1
          done

      - name: Install Just
        if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu')
        uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # v2
        with:
          tool: just

      - name: Install cargo nextest
        uses: taiki-e/install-action@98ec31d284eb962f41c14065e9391a955aa810cf # nextest

      - name: Install Rust toolchain
        run: |
          rustup update --no-self-update ${{ env.RUST_CHANNEL }}
          rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src clippy
          rustup default ${{ env.RUST_CHANNEL }}

      - name: Cache Dependencies
        uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2
        with:
          # workspaces: "rust -> target"
          key: ${{ env.RUST_CHANNEL }}

      - name: Run tests with Docker services
        if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu')
        env:
          RUST_LOG: icechunk=trace
          R2_BUCKET: ${{ secrets.R2_BUCKET }}
          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}

          AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

          TIGRIS_BUCKET: ${{ secrets.TIGRIS_BUCKET }}
          TIGRIS_REGION: ${{ secrets.TIGRIS_REGION }}
          TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }}
          TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }}

        run: |
          just profile=ci compile-tests "--locked"
          just profile=ci test

      - name: Run doc tests
        if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu')
        run: |
          just profile=ci doctest

      - name: Run examples
        if: contains(matrix.os, 'linux') || contains(matrix.os, 'ubuntu')
        run: |
          just profile=ci run-all-examples

      - name: Run unit tests only
        if: ${{ !contains(matrix.os, 'linux') && !contains(matrix.os, 'ubuntu') }}
        run: |
          cargo test --lib --profile ci

      - name: Run integration tests against object stores
        if: github.event_name == 'cron'
        env:
          R2_BUCKET: ${{ secrets.R2_BUCKET }}
          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}

          AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

          TIGRIS_BUCKET: ${{ secrets.TIGRIS_BUCKET }}
          TIGRIS_REGION: ${{ secrets.TIGRIS_REGION }}
          TIGRIS_ACCESS_KEY_ID: ${{ secrets.TIGRIS_ACCESS_KEY_ID }}
          TIGRIS_SECRET_ACCESS_KEY: ${{ secrets.TIGRIS_SECRET_ACCESS_KEY }}

        run: |
          cargo test --profile ci --workspace --all-targets -- --ignored

  # shuttle:
  #   name: Shuttle concurrency tests
  #   timeout-minutes: 30
  #   runs-on: ubuntu-latest
  #   steps:
  #     - name: Checkout repository
  #       uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
  #       with:
  #         ref: ${{ github.event.pull_request.head.sha }}

  #     - name: Install Rust toolchain
  #       run: |
  #         rustup update --no-self-update ${{ env.RUST_CHANNEL }}
  #         rustup default ${{ env.RUST_CHANNEL }}

  #     - name: Cache Dependencies
  #       uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2
  #       with:
  #         key: shuttle-${{ env.RUST_CHANNEL }}

  #     - name: Compile shuttle tests
  #       run: cargo test -p icechunk --features shuttle --test test_shuttle --no-run

  #     - name: Run shuttle tests
  #       run: cargo test -p icechunk --features shuttle --test test_shuttle -- --nocapture

  wasm-build:
    name: WASM Build
    timeout-minutes: 15
    runs-on: runs-on=${{ github.run_id }}/runner=8cpu-linux-x64/extras=s3-cache
    defaults:
      run:
        working-directory: ./
    env:
      # Keep this as a compile smoke test; avoid failing on existing warnings in
      # no-default-features wasm cfg combinations.
      RUSTFLAGS: ""

    steps:
      - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Install Rust toolchain
        run: |
          rustup update --no-self-update ${{ env.RUST_CHANNEL }}
          rustup default ${{ env.RUST_CHANNEL }}
          rustup target add wasm32-wasip1-threads

      - name: Install LLVM toolchain for wasm C deps
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends clang llvm wasi-libc libc++-dev

      - name: Show wasm clang include search paths
        run: |
          echo | clang --target=wasm32-wasi -E -x c - -v
          echo '#include <limits.h>' | clang --target=wasm32-wasi -E -x c - -v > /dev/null
          ls -la /usr/include/wasm32-wasi || true
          ls -la /usr/lib/wasm32-wasi || true
          test -f /usr/include/wasm32-wasi/limits.h && echo "found /usr/include/wasm32-wasi/limits.h"

      - name: Cache Dependencies
        uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2
        with:
          key: wasm-${{ env.RUST_CHANNEL }}

      - name: Run wasm build check
        env:
          CC_wasm32_wasip1_threads: clang
          CXX_wasm32_wasip1_threads: clang++
          AR_wasm32_wasip1_threads: llvm-ar
          CC_wasm32_wasi: clang
          CXX_wasm32_wasi: clang++
          AR_wasm32_wasi: llvm-ar
          WASI_SYSROOT: /usr
          CFLAGS_wasm32_wasip1_threads: --sysroot=/usr -isystem /usr/include/wasm32-wasi
          CXXFLAGS_wasm32_wasip1_threads: --sysroot=/usr -isystem /usr/include/wasm32-wasi
          CFLAGS_wasm32_wasi: --sysroot=/usr -isystem /usr/include/wasm32-wasi
          CXXFLAGS_wasm32_wasi: --sysroot=/usr -isystem /usr/include/wasm32-wasi
        run: |
          cargo build -p icechunk --no-default-features --target wasm32-wasip1-threads

      - name: Run no-default-features lib tests (proxy for wasm feature set)
        run: |
          cargo test -p icechunk --no-default-features --lib