perf: Disable morsel splitting for fast-count on streaming engine #35591
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Python | |
| on: | |
| pull_request: | |
| paths: | |
| - Cargo.lock | |
| - py-polars/** | |
| - docs/source/src/python/** | |
| - crates/** | |
| - .github/workflows/test-python.yml | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - Cargo.lock | |
| - crates/** | |
| - docs/source/src/python/** | |
| - py-polars/** | |
| - .github/workflows/test-python.yml | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| RUSTFLAGS: -C debuginfo=0 # Do not produce debug symbols to keep memory usage down | |
| RUST_BACKTRACE: 1 | |
| PYTHONUTF8: 1 | |
| defaults: | |
| run: | |
| working-directory: py-polars | |
| shell: bash | |
| jobs: | |
| test-python: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ['3.10', '3.12', '3.13', '3.14', '3.14t'] | |
| include: | |
| - os: windows-latest | |
| python-version: '3.14' | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Free some disk space (remove unused packages) | |
| uses: jlumbroso/free-disk-space@v1.3.1 | |
| if: ${{ matrix.os != 'windows-latest' }} | |
| - name: Enable long paths on Windows | |
| if: matrix.os == 'windows-latest' | |
| shell: | |
| powershell | |
| run: | | |
| Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -Value 1 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Graphviz | |
| uses: ts-graphviz/setup-graphviz@v2 | |
| - name: Create virtual environment | |
| env: | |
| BIN: ${{ matrix.os == 'windows-latest' && 'Scripts' || 'bin' }} | |
| run: | | |
| python -m venv .venv | |
| echo "$GITHUB_WORKSPACE/py-polars/.venv/$BIN" >> $GITHUB_PATH | |
| echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/py-polars/.venv" >> $GITHUB_ENV | |
| echo "VENV=$GITHUB_WORKSPACE/py-polars/.venv" >> $GITHUB_ENV | |
| - name: Install maturin | |
| if: matrix.python-version == '3.14t' | |
| run: | | |
| pip install uv | |
| uv pip install maturin | |
| - name: Install Python dependencies | |
| if: matrix.python-version != '3.14t' | |
| run: | | |
| pip install uv | |
| # Install typing-extensions separately whilst the `--extra-index-url` in `requirements-ci.txt` | |
| # doesn't have an up-to-date typing-extensions, see | |
| # https://github.com/astral-sh/uv/issues/6028#issuecomment-2287232150 | |
| uv pip install -U typing-extensions | |
| uv pip install --compile-bytecode -r requirements-dev.txt -r requirements-ci.txt --verbose --index-strategy=unsafe-best-match | |
| - name: Set up Rust | |
| run: rustup show | |
| - name: Cache Rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: py-polars | |
| save-if: ${{ github.ref_name == 'main' }} | |
| - name: Install Polars | |
| run: | | |
| # Make sure we use the Polars we're about to build. | |
| uv pip uninstall polars-runtime-32 polars-runtime-64 polars-runtime-compat | |
| pip install --no-deps -e . | |
| maturin develop --manifest-path runtime/polars-runtime-32/Cargo.toml | |
| - name: Run doctests | |
| if: github.ref_name != 'main' && matrix.python-version == '3.14' && matrix.os == 'ubuntu-latest' | |
| run: | | |
| python tests/docs/run_doctest.py | |
| pytest tests/docs/test_user_guide.py -m docs | |
| - name: Run tests | |
| if: github.ref_name != 'main' && matrix.python-version != '3.14t' | |
| env: | |
| POLARS_TIMEOUT_MS: 60000 | |
| run: pytest -n auto -m "not release and not benchmark and not docs" | |
| - name: Run tests with new streaming engine | |
| if: github.ref_name != 'main' && matrix.python-version != '3.14t' | |
| env: | |
| POLARS_AUTO_NEW_STREAMING: 1 | |
| POLARS_TIMEOUT_MS: 60000 | |
| run: pytest -n auto -m "not may_fail_auto_streaming and not slow and not write_disk and not release and not docs and not hypothesis and not benchmark and not ci_only" | |
| - name: Run tests async reader tests | |
| if: github.ref_name != 'main' && matrix.os != 'windows-latest' && matrix.python-version != '3.14t' | |
| env: | |
| POLARS_FORCE_ASYNC: 1 | |
| POLARS_TIMEOUT_MS: 60000 | |
| run: pytest -n auto -m "not release and not benchmark and not docs" tests/unit/io/ | |
| - name: Run tests multiscan force empty capabilities | |
| if: github.ref_name != 'main' && matrix.python-version != '3.14t' | |
| env: | |
| POLARS_FORCE_EMPTY_READER_CAPABILITIES: 1 | |
| POLARS_TIMEOUT_MS: 60000 | |
| run: | | |
| pytest -n auto -m "not release and not benchmark and not docs" tests/unit/io/test_multiscan.py | |
| pytest -n auto -m "not release and not benchmark and not docs" tests/unit/io/test_scan_row_deletion.py | |
| pytest -n auto -m "not release and not benchmark and not docs" tests/unit/io/test_iceberg.py | |
| - name: Check import without optional dependencies | |
| if: github.ref_name != 'main' && matrix.os == 'ubuntu-latest' && (matrix.python-version == '3.14' || matrix.python-version == '3.14t') | |
| run: | | |
| declare -a deps=("pandas" | |
| "pyarrow" | |
| "fsspec" | |
| "matplotlib" | |
| "backports.zoneinfo" | |
| "connectorx" | |
| "pyiceberg" | |
| "deltalake" | |
| "xlsx2csv" | |
| ) | |
| for d in "${deps[@]}" | |
| do | |
| echo "uninstall $i and check imports..." | |
| pip uninstall "$d" -y | |
| python -c 'import polars' | |
| done |