rust.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Rust

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

on:
  push:
    paths-ignore:
      - "docs/**"
      - "**.md"
      - ".github/ISSUE_TEMPLATE/**"
      - ".github/pull_request_template.md"
  pull_request:
    paths-ignore:
      - "docs/**"
      - "**.md"
      - ".github/ISSUE_TEMPLATE/**"
      - ".github/pull_request_template.md"
  # manual trigger
  # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
  workflow_dispatch:

jobs:
  # Check crate compiles and base cargo check passes
  linux-build-lib:
    name: linux build test
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v5
        with:
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
      - name: Prepare cargo build
        run: |
          # Adding `--locked` here to assert that the `Cargo.lock` file is up to
          # date with the manifest. When this fails, please make sure to commit
          # the changes to `Cargo.lock` after building with the updated manifest.
          cargo check --profile ci --workspace --all-targets --locked

  linux-test:
    name: test linux crates
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v5
        with:
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
      - name: Run tests
        run: |
          export PATH=$PATH:$HOME/d/protoc/bin
          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
          cargo test --profile ci --features=testcontainers
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  # Run `cargo doc` to ensure the rustdoc is clean
  linux-rustdoc:
    name: cargo doc
    needs: linux-build-lib
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v5
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: stable
      - name: Run cargo doc
        run: ci/scripts/rust_docs.sh

  linux-build-workspace:
    name: check linux workspace
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v5
        with:
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
      - name: Try to compile when `--no-default-features` is selected
        run: |
          export PATH=$PATH:$HOME/d/protoc/bin
          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
          cargo check --profile ci -p ballista-scheduler -p ballista-executor -p ballista-core -p ballista --no-default-features --locked
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  ballista-test:
    name: test linux balista
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v5
        with:
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
      - name: Run Ballista tests
        run: |
          export PATH=$PATH:$HOME/d/protoc/bin
          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
          cd ballista
          # Ensure also compiles in standalone mode
          cargo test --profile ci --no-default-features --features standalone --locked
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  windows-test:
    name: windows test
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v5
        with:
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-windows-builder
      - name: Run tests
        shell: bash
        run: |
          export PATH=$PATH:$HOME/d/protoc/bin
          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
          export TMP=$(pwd)/.temp
          mkdir -p "$TMP"
          cargo test --profile ci --locked

  macos-test:
    name: macos test
    runs-on: macos-latest
    steps:
      - uses: actions/checkout@v5
        with:
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-macos-builder
      - name: Run tests
        shell: bash
        run: |
          export PATH=$PATH:$HOME/d/protoc/bin
          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
          cargo test --profile ci --locked

  # verify that the benchmark queries return the correct results
  verify-benchmark-results:
    name: verify benchmark results (amd64)
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v5
        with:
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
      - name: Verify that benchmark queries return expected results
        run: |
          cargo test --package ballista-benchmarks --profile release-nonlto --features=ci --locked -- --test-threads=1

  lint:
    name: Lint
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v5
      - name: Setup toolchain
        run: |
          rustup toolchain install stable
          rustup default stable
          rustup component add rustfmt
      - name: Run
        run: ci/scripts/rust_fmt.sh
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  clippy:
    name: Clippy
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [stable]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v5
        with:
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{ matrix.rust }}
      - name: Install Clippy
        run: |
          rustup component add clippy
      - name: Run clippy
        run: ci/scripts/rust_clippy.sh
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  cargo-toml-formatting-checks:
    name: Check Cargo.toml formatting
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [stable]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v5
        with:
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: ${{ matrix.rust }}
      - name: Install cargo-tomlfmt
        run: |
          which cargo-tomlfmt || cargo install cargo-tomlfmt
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"
      - name: Check Cargo.toml formatting
        run: |
          # if you encounter error, try rerun the command below, finally run 'git diff' to
          # check which Cargo.toml introduces formatting violation
          #
          # ignore ./Cargo.toml because putting workspaces in multi-line lists make it easy to read
          ci/scripts/rust_toml_fmt.sh
          if test -f "./Cargo.toml.bak"; then
              echo "cargo tomlfmt found format violations"
              exit 1
          fi
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"
# Coverage job was failing. https://github.com/apache/arrow-datafusion/issues/590 tracks re-instating it

# coverage:
#   name: Coverage
#   runs-on: ubuntu-latest
#   strategy:
#     matrix:
#       arch: [amd64]
#       rust: [stable]
#   steps:
#     - uses: actions/checkout@v5
#       with:
#         submodules: true
#     - name: Cache Cargo
#       uses: actions/cache@v4
#       with:
#         path: /home/runner/.cargo
#         # this key is not equal because the user is different than on a container (runner vs github)
#         key: cargo-coverage-cache-
#     - name: Cache Rust dependencies
#       uses: actions/cache@v4
#       with:
#         path: /home/runner/target
#         # this key is not equal because coverage uses different compilation flags.
#         key: ${{ runner.os }}-${{ matrix.arch }}-target-coverage-cache-${{ matrix.rust }}-
#     - name: Run coverage
#       run: |
#         export ARROW_TEST_DATA=$(pwd)/testing/data
#         export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data

#         # 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0
#         # see https://github.com/xd009642/tarpaulin/issues/618
#         cargo install --version 0.16.0 cargo-tarpaulin
#         cargo tarpaulin --out Xml
#       env:
#         CARGO_HOME: "/home/runner/.cargo"
#         CARGO_TARGET_DIR: "/home/runner/target"
#     - name: Report coverage
#       continue-on-error: true
#       run: bash <(curl -s https://codecov.io/bash)