Keep hffs cache in workers when streaming #6019
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| - ci-* | |
| env: | |
| CI_HEADERS: ${{ secrets.CI_HEADERS }} | |
| jobs: | |
| check_code_quality: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.9" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install .[quality] | |
| - name: Check quality | |
| run: | | |
| ruff check tests src benchmarks utils setup.py # linter | |
| ruff format --check tests src benchmarks utils setup.py # formatter | |
| test: | |
| needs: check_code_quality | |
| strategy: | |
| matrix: | |
| test: ['unit', 'integration'] | |
| os: [ubuntu-latest, windows-latest] | |
| deps_versions: [deps-latest, deps-minimum] | |
| continue-on-error: ${{ matrix.test == 'integration' }} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup FFmpeg | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: | | |
| sudo apt update | |
| sudo apt install -y ffmpeg | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.9" | |
| - name: Setup conda env (windows) | |
| if: ${{ matrix.os == 'windows-latest' }} | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: true | |
| miniconda-version: "latest" | |
| activate-environment: test | |
| python-version: "3.9" | |
| - name: Setup FFmpeg (windows) | |
| if: ${{ matrix.os == 'windows-latest' }} | |
| run: conda install "ffmpeg=7.0.1" -c conda-forge | |
| - name: Upgrade pip | |
| run: python -m pip install --upgrade pip | |
| - name: Install uv | |
| run: pip install --upgrade uv | |
| - name: Install dependencies | |
| run: uv pip install --system "datasets[tests] @ ." | |
| - name: Install dependencies (latest versions) | |
| if: ${{ matrix.deps_versions == 'deps-latest' }} | |
| run: uv pip install --system --upgrade pyarrow huggingface-hub "dill<0.3.9" | |
| - name: Install dependencies (minimum versions) | |
| if: ${{ matrix.deps_versions != 'deps-latest' }} | |
| run: uv pip install --system pyarrow==21.0.0 huggingface-hub==0.25.0 transformers dill==0.3.1.1 | |
| - name: Print dependencies | |
| run: uv pip list | |
| - name: Test with pytest | |
| run: | | |
| python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/ | |
| test_py312: | |
| needs: check_code_quality | |
| strategy: | |
| matrix: | |
| test: ['unit'] | |
| os: [ubuntu-latest, windows-latest] | |
| deps_versions: [deps-latest] | |
| continue-on-error: false | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup FFmpeg | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: | | |
| sudo apt update | |
| sudo apt install -y ffmpeg | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Setup conda env (windows) | |
| if: ${{ matrix.os == 'windows-latest' }} | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: true | |
| miniconda-version: "latest" | |
| activate-environment: test | |
| python-version: "3.12" | |
| - name: Setup FFmpeg (windows) | |
| if: ${{ matrix.os == 'windows-latest' }} | |
| run: conda install "ffmpeg=7.0.1" -c conda-forge | |
| - name: Upgrade pip | |
| run: python -m pip install --upgrade pip | |
| - name: Install uv | |
| run: pip install --upgrade uv | |
| - name: Install dependencies | |
| run: uv pip install --system "datasets[tests] @ ." | |
| - name: Print dependencies | |
| run: uv pip list | |
| - name: Test with pytest | |
| run: | | |
| python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/ | |
| test_py312_future: | |
| needs: check_code_quality | |
| strategy: | |
| matrix: | |
| test: ['unit'] | |
| os: [ubuntu-latest, windows-latest] | |
| deps_versions: [deps-latest] | |
| continue-on-error: false | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup FFmpeg | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: | | |
| sudo apt update | |
| sudo apt install -y ffmpeg | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Setup conda env (windows) | |
| if: ${{ matrix.os == 'windows-latest' }} | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: true | |
| miniconda-version: "latest" | |
| activate-environment: test | |
| python-version: "3.12" | |
| - name: Setup FFmpeg (windows) | |
| if: ${{ matrix.os == 'windows-latest' }} | |
| run: conda install "ffmpeg=7.0.1" -c conda-forge | |
| - name: Upgrade pip | |
| run: python -m pip install --upgrade pip | |
| - name: Install uv | |
| run: pip install --upgrade uv | |
| - name: Install dependencies | |
| run: | | |
| uv pip install --system "datasets[tests_numpy2] @ ." | |
| # TODO: remove once transformers v5 / huggingface_hub v1 are released officially | |
| uv pip uninstall --system transformers huggingface_hub | |
| uv pip install --system --prerelease=allow git+https://github.com/huggingface/transformers.git | |
| - name: Print dependencies | |
| run: pip list | |
| - name: Test with pytest | |
| run: | | |
| python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/ |