ci(datasets): Migrate Flower Datasets to uv

.github/workflows/datasets-e2e.yml

@@ -18,6 +18,8 @@ concurrency:
 
 env:
   FLWR_TELEMETRY_ENABLED: 0
+  UV_NO_MANAGED_PYTHON: 1
+  UV_PYTHON_DOWNLOADS: never
 
 jobs:
   frameworks:
@@ -46,6 +48,11 @@ jobs:
         uses: ./.github/actions/bootstrap
         with:
           python-version: 3.10.19
+          poetry-skip: "true"
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "0.9.11"
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@v1.3.1
         with:
@@ -61,6 +68,6 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y ffmpeg
       - name: Install dependencies
-        run: python -m poetry install
+        run: uv sync --frozen
       - name: Run tests
-        run: python -m unittest discover -p '*_test.py'
+        run: uv run python -m unittest discover -p '*_test.py'

.github/workflows/datasets.yml

@@ -22,6 +22,8 @@ concurrency:
 
 env:
   FLWR_TELEMETRY_ENABLED: 0
+  UV_NO_MANAGED_PYTHON: 1
+  UV_PYTHON_DOWNLOADS: never
 
 jobs:
   test_core:
@@ -44,6 +46,11 @@ jobs:
         uses: ./.github/actions/bootstrap
         with:
           python-version: ${{ matrix.python }}
+          poetry-skip: "true"
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "0.9.11"
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@v1.3.1
         with:
@@ -58,10 +65,10 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y ffmpeg
-      - name: Install dependencies (mandatory only)
+      - name: Install dependencies
         run: |
           cd datasets
-          python -m poetry install --all-extras
+          uv sync --frozen --all-extras
       - name: Cache Hugging Face datasets
         uses: actions/cache@v3
         with:
@@ -70,11 +77,14 @@ jobs:
           restore-keys: hf-datasets-
       - name: Set Hugging Face token
         run: |
+          cd datasets
           if [ -n "${{ secrets.HF_TOKEN }}" ]; then
             echo "Logging into Hugging Face..."
-            hf auth login --token ${{ secrets.HF_TOKEN }}
+            uv run hf auth login --token ${{ secrets.HF_TOKEN }}
           else
             echo "Skipping Hugging Face login stage (HF_TOKEN not set)"
           fi
       - name: Test (formatting + unit tests)
-        run: ./datasets/dev/test.sh
+        run: |
+          cd datasets
+          uv run ./dev/test.sh

.github/workflows/framework-docs.yml

@@ -32,8 +32,8 @@ jobs:
       - name: Install Flower Framework and Flower Datasets with dependencies
         run: |
           cd framework
-          python -m poetry add ../datasets
           python -m poetry install
+          python -m pip install -e ../datasets
       - name: Build docs
         run: ./dev/build-docs.sh ${{ github.ref == 'refs/heads/main' && github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork }}
       - name: Deploy docs

datasets/README.md

@@ -20,6 +20,73 @@ For a complete installation guide visit the [Flower Datasets Documentation](http
 pip install flwr-datasets[vision]
 ```
 
+> [!NOTE]
+> The `audio` extra currently supports Python `<3.12` due to upstream dependency
+> constraints.
+
+## Development (uv)
+
+Flower Datasets uses `uv` for development and CI.
+
+### Setup
+
+```bash
+cd datasets
+uv sync --all-extras
+```
+
+> [!TIP]
+> Use `uv sync --frozen --all-extras` to ensure `uv.lock` is not modified.
+
+### Run checks (formatting + unit tests)
+
+```bash
+cd datasets
+uv run ./dev/test.sh
+```
+
+### Format
+
+```bash
+cd datasets
+uv run ./dev/format.sh
+```
+
+### Build docs
+
+```bash
+cd datasets
+uv run ./dev/build-flwr-datasets-docs.sh
+```
+
+### Run E2E tests
+
+```bash
+cd datasets/e2e/pytorch
+uv sync --frozen
+uv run python -m unittest discover -p '*_test.py'
+```
+
+Repeat for `datasets/e2e/scikit-learn` and `datasets/e2e/tensorflow`.
+
+### Dependency management (no `uv pip`)
+
+```bash
+cd datasets
+
+# Add a runtime dependency
+uv add <package>
+
+# Add a dev dependency
+uv add --dev <package>
+
+# Add a dependency to an extra (e.g. "vision")
+uv add --optional vision <package>
+
+# Update lockfile (commit the result)
+uv lock
+```
+
 ## Overview
 
 Flower Datasets library supports:

datasets/dev/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -e
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../
+
+uv build --clear

datasets/dev/publish.sh

@@ -18,4 +18,4 @@
 set -e
 cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../
 
-python -m poetry publish -u __token__ -p ${PYPI_TOKEN}
+uv publish --token "${PYPI_TOKEN}"

datasets/docs/source/contributor-how-to-develop.rst

@@ -0,0 +1,82 @@
+How to develop Flower Datasets
+==============================
+
+Flower Datasets uses `uv <https://docs.astral.sh/uv/>`_ for development and CI.
+
+Setup
+-----
+
+Install dependencies (including all extras):
+
+.. code-block:: bash
+
+  cd datasets
+  uv sync --all-extras
+
+.. tip::
+
+  Use ``uv sync --frozen --all-extras`` to ensure ``uv.lock`` is not modified.
+
+Run checks
+----------
+
+Run formatting and unit tests:
+
+.. code-block:: bash
+
+  cd datasets
+  uv run ./dev/test.sh
+
+Format
+------
+
+.. code-block:: bash
+
+  cd datasets
+  uv run ./dev/format.sh
+
+Build docs
+----------
+
+.. code-block:: bash
+
+  cd datasets
+  uv run ./dev/build-flwr-datasets-docs.sh
+
+Run E2E tests
+-------------
+
+Each E2E directory is a standalone `uv` project.
+
+.. code-block:: bash
+
+  cd datasets/e2e/pytorch
+  uv sync --frozen
+  uv run python -m unittest discover -p '*_test.py'
+
+Repeat for ``datasets/e2e/scikit-learn`` and ``datasets/e2e/tensorflow``.
+
+Dependency management
+---------------------
+
+Avoid using ``uv pip``.
+
+.. code-block:: bash
+
+  cd datasets
+
+  # Add a runtime dependency
+  uv add <package>
+
+  # Add a dev dependency
+  uv add --dev <package>
+
+  # Add a dependency to an extra (e.g. "vision")
+  uv add --optional vision <package>
+
+After changing dependencies, update the lockfile and commit it:
+
+.. code-block:: bash
+
+  cd datasets
+  uv lock

datasets/docs/source/how-to-install-flwr-datasets.rst

@@ -4,7 +4,7 @@ Installation
 Python Version
 --------------
 
-Flower Datasets requires `Python 3.9 <https://docs.python.org/3.9/>`_ or above.
+Flower Datasets requires `Python 3.10 <https://docs.python.org/3.10/>`_ or above.
 
 
 Install stable release (pip)
@@ -28,6 +28,11 @@ For audio datasets (e.g. Speech Command) ``flwr-datasets`` should be installed w
 
   python -m pip install "flwr-datasets[audio]"
 
+.. note::
+
+  The ``audio`` extra currently supports Python ``<3.12`` due to upstream dependency
+  constraints.
+
 Install directly from GitHub (pip)
 ----------------------------------
 
@@ -70,4 +75,3 @@ If everything works, it should print the version of Flower Datasets to the comma
 .. code-block:: none
 
   0.5.0
-

datasets/docs/source/index.rst

@@ -70,6 +70,7 @@ Information-oriented API reference and other reference material.
    :maxdepth: 1
    :caption: Contributor tutorials
 
+   contributor-how-to-develop
    contributor-how-to-contribute-dataset
 
 

datasets/e2e/pytorch/pyproject.toml

@@ -1,17 +1,14 @@
-[build-system]
-requires = ["poetry-core>=2.1.3"]
-build-backend = "poetry.core.masonry.api"
-
-[tool.poetry]
+[project]
 name = "fds-e2e-pytorch"
 version = "0.1.0"
 description = "Flower Datasets with PyTorch"
-authors = ["The Flower Authors <hello@flower.ai>"]
-package-mode = false
+requires-python = ">=3.10"
+dependencies = [
+    "flwr-datasets[vision]",
+    "torch>=1.12.0,<3.0.0",
+    "torchvision>=0.19.0,<1.0.0",
+    "parameterized==0.9.0",
+]
 
-[tool.poetry.dependencies]
-python = "^3.10"
-flwr-datasets = { path = "./../../", extras = ["vision"] }
-torch = ">=1.12.0,<3.0.0"
-torchvision = ">=0.19.0,<1.0.0"
-parameterized = "==0.9.0"
+[tool.uv.sources]
+flwr-datasets = { path = "../..", editable = true }