main.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http:/www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Python Tests
on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

env:
  MATURIN_PEP517_ARGS: --profile=dev

jobs:
  validate-tpch:
    runs-on: ${{ matrix.platform.image }}
    strategy:
      fail-fast: false
      matrix:
        platform:
          - image: ubuntu-latest
            target: x86_64-unknown-linux-musl
        python-version:
          - "3.10"
          - "3.11"
          - "3.12"
        ray-version:
          - "2.40"
          - "2.41"
          - "2.42.1"
          - "2.43"
    steps:
      - uses: actions/checkout@v4

      - name: Setup Rust Toolchain
        uses: dtolnay/rust-toolchain@stable
        id: rust-toolchain
        with:
          target: ${{ matrix.platform.target }}

      - name: Check Rust output
        id: rust-toolchain-out
        run: |
          rustc --version --verbose
          cargo --version

      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          version: "27.4"
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Cache Cargo
        uses: actions/cache@v4
        with:
          path: ~/.cargo
          key: cargo-cache-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('Cargo.lock') }}

      - name: install uv
        uses: astral-sh/setup-uv@v5
        with:
          enable-cache: true
          python-version: ${{ matrix.python-version }}

      - name: initial project sync
        run: |
          cargo --version
          uv sync --no-install-package datafusion-ray --no-install-package ray

      # reset the version of ray in pyproject.toml
      # to agree align with our matrix version
      - name: add ray
        run: |
          cargo --version
          uv add --no-sync 'ray[default]==${{ matrix.ray-version}}'

      # sync the environment, building everything other than datafusion-ray
      - name: install dependencies
        run: |
          uv sync --no-install-package datafusion-ray
        working-directory: ${{ github.workspace }}

      # build datafusion ray honoring the MATURIN_PEP517_ARGS env
      # var to ensure that we build this in dev mode so its faster
      - name: maturin develop
        run: uv run maturin develop --uv
        working-directory: ${{ github.workspace }}

      - name: Cache the generated dataset
        id: cache-tpch-dataset
        uses: actions/cache@v4
        with:
          path: ./testdata/tpch
          key: tpch-data

      - name: create the dataset
        if: ${{ steps.cache-tpch-dataset.outputs.cache-hit != 'true' }}
        run: |
          uv run tpch/make_data.py 1 testdata/tpch/

      # run the tpcbench.py file with --validate which will cause
      # it to exit with an error code if any of the queries do not validate
      - name: validate tpch
        env:
          DATAFUSION_RAY_LOG_LEVEL: debug
          RAY_COLOR_PREFIX: 1
          RAY_DEDUP_LOGS: 0
          RUST_BACKTRACE: 1
        run: |
          uv run python tpch/tpcbench.py \
            --data='file:///${{ github.workspace }}/testdata/tpch/' \
            --concurrency 3 \
            --partitions-per-processor 2 \
            --batch-size=8192 \
            --processor-pool-min=20 \
            --validate