ci(*:skip): Move all Python scripts under dev into devtool

.devcontainer/post-create.sh

@@ -24,7 +24,7 @@ for f in "${files[@]}"; do
   comment_taplo "$f"
 done
 
-sudo poetry install --with dev --extras "simulation"
+sudo poetry install --all-extras
 
 # Restore taplo lines in "files"
 for f in "${files[@]}"; do

.github/workflows/build-deploy-non-framework-docs.yml

@@ -35,11 +35,11 @@ jobs:
           python -m poetry install
           python -m pip install -e ../datasets
       - name: Update HTML theme options
-        run: python dev/update-html-themes.py
+        run: PYTHONPATH=dev python -m devtool.update_html_themes
       - name: Build baselines docs
         run: ./dev/build-baseline-docs.sh
       - name: Build examples docs
-        run: python dev/build-example-docs.py
+        run: PYTHONPATH=dev python -m devtool.build_example_docs
       - name: Build datasets docs
         run: ./datasets/dev/build-flwr-datasets-docs.sh
       - name: Deploy docs

.github/workflows/datasets-test.yml

@@ -6,12 +6,14 @@ on:
       - main
     paths:
       - "datasets/**"
+      - "dev/devtool/**"
       - ".github/workflows/datasets-*.yml"
   pull_request:
     branches:
       - main
     paths:
       - "datasets/**"
+      - "dev/devtool/**"
       - ".github/workflows/datasets-*.yml"
 
 concurrency:

.github/workflows/framework-docs-update-translations.yml

@@ -32,7 +32,6 @@ jobs:
         run: |
           cd framework
           python -m poetry install
-          pip install sphinx==7.4.7
 
       - name: Install pandoc
         uses: nikeee/setup-pandoc@v1

.github/workflows/framework-e2e.yml

@@ -44,7 +44,7 @@ jobs:
       - uses: actions/checkout@v5
       - name: Bootstrap
         uses: ./.github/actions/bootstrap
-      - name: Install dependencies (mandatory only)
+      - name: Install dependencies
         run: |
           cd framework
           python -m poetry install --all-extras
@@ -225,7 +225,8 @@ jobs:
       - name: Install Flower wheel from artifact store
         if: ${{ needs.changes.outputs.framework == 'true' && github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
         run: |
-          python -m pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
+          WHEEL_URL="https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}"
+          python -m pip install "flwr[simulation] @ ${WHEEL_URL}"
       - name: Download dataset
         if: ${{ needs.changes.outputs.framework == 'true' && matrix.dataset }}
         run: python -c "${{ matrix.dataset }}"
@@ -284,11 +285,12 @@ jobs:
       - name: Install dependencies
         if: ${{ needs.changes.outputs.framework == 'true' }}
         run: |
-          python -m poetry install
+          python -m poetry install --only main
       - name: Install Flower wheel from artifact store
         if: ${{ needs.changes.outputs.framework == 'true' && github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
         run: |
-          python -m pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
+          WHEEL_URL="https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}"
+          python -m pip install "flwr[simulation] @ ${WHEEL_URL}"
       - name: Cache Datasets
         if: ${{ needs.changes.outputs.framework == 'true' }}
         uses: actions/cache@v4
@@ -335,7 +337,8 @@ jobs:
       - name: Install Flower wheel from artifact store
         if: ${{ needs.changes.outputs.framework == 'true' && github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
         run: |
-          python -m pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
+          WHEEL_URL="https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}"
+          python -m pip install "flwr[simulation] @ ${WHEEL_URL}"
       - name: Install project dependencies
         if: ${{ needs.changes.outputs.framework == 'true' }}
         working-directory: examples/${{ matrix.example }}
@@ -378,7 +381,8 @@ jobs:
       - name: Install Flower wheel from artifact store
         if: ${{ needs.changes.outputs.framework == 'true' && github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
         run: |
-          python -m pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
+          WHEEL_URL="https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}"
+          python -m pip install "flwr[simulation] @ ${WHEEL_URL}"
       - name: Create project and install it
         if: ${{ needs.changes.outputs.framework == 'true' }}
         run: |
@@ -424,7 +428,8 @@ jobs:
       - name: Install Flower wheel from artifact store
         if: ${{ needs.changes.outputs.framework == 'true' && github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
         run: |
-          python -m pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
+          WHEEL_URL="https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}"
+          python -m pip install "flwr[simulation] @ ${WHEEL_URL}"
       - name: Create project, build, and install it
         if: ${{ needs.changes.outputs.framework == 'true' }}
         run: |

.github/workflows/framework-test.yml

@@ -33,6 +33,7 @@ jobs:
           filters: |
             framework:
               - 'framework/**/*'
+              - 'dev/devtool/**'
               - '.github/workflows/framework-test.yml'
             ex_bench:
               - 'examples/**/*'
@@ -61,7 +62,7 @@ jobs:
         uses: ./.github/actions/bootstrap
         with:
           python-version: ${{ matrix.python }}
-      - name: Install dependencies (mandatory only)
+      - name: Install dependencies
         if: ${{ needs.changes.outputs.framework == 'true' || needs.changes.outputs.ex_bench == 'true' }}
         run: |
           cd framework 

.github/workflows/repo-check-pr-title.yml

@@ -24,4 +24,4 @@ jobs:
       - name: Check PR title format 
         env:
           PR_TITLE: ${{ github.event.pull_request.title }}
-        run: python ./dev/check_pr_title.py "$PR_TITLE"
+        run: PYTHONPATH=dev python -m devtool.check_pr_title "$PR_TITLE"

dev/changelog_config.toml

@@ -1,4 +1,4 @@
-# Defines the PR title format to be used in the `check_pr_title.py` script
+# Defines the PR title format to be used in the `devtool.check_pr_title` script
 # and in the `update_changelog.py` script
 
 type = ["ci", "docs", "feat", "fix", "refactor", "break"]

dev/build-example-docs.py → dev/devtool/build_example_docs.py

@@ -20,8 +20,8 @@
 import subprocess
 from pathlib import Path
 
-ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
-INDEX = os.path.join(ROOT, "examples", "docs", "source", "index.rst")
+ROOT = Path(__file__).resolve().parents[2]
+INDEX = ROOT / "examples" / "docs" / "source" / "index.rst"
 
 initial_text = """
 Flower Examples Documentation
@@ -179,14 +179,14 @@ def _copy_markdown_files(example):
         if file.endswith(".md"):
             src = os.path.join(example, file)
             dest = os.path.join(
-                ROOT, "examples", "docs", "source", os.path.basename(example) + ".md"
+                str(ROOT), "examples", "docs", "source", os.path.basename(example) + ".md"
             )
             shutil.copyfile(src, dest)
 
 
 def _add_gh_button(example):
     gh_text = f'[<img src="_static/view-gh.png" alt="View on GitHub" width="200"/>](https://github.com/adap/flower/blob/main/examples/{example})'
-    readme_file = os.path.join(ROOT, "examples", "docs", "source", example + ".md")
+    readme_file = os.path.join(str(ROOT), "examples", "docs", "source", example + ".md")
     with open(readme_file, "r+") as f:
         content = f.read()
         if gh_text not in content:
@@ -200,7 +200,7 @@ def _add_gh_button(example):
 
 def _copy_images(example):
     static_dir = os.path.join(example, "_static")
-    dest_dir = os.path.join(ROOT, "examples", "docs", "source", "_static")
+    dest_dir = os.path.join(str(ROOT), "examples", "docs", "source", "_static")
     if os.path.isdir(static_dir):
         for file in os.listdir(static_dir):
             if file.endswith((".jpg", ".png", ".jpeg")):
@@ -220,10 +220,10 @@ def _add_all_entries():
 
 
 def _main():
-    if os.path.exists(INDEX):
-        os.remove(INDEX)
+    if INDEX.exists():
+        INDEX.unlink()
 
-    with open(INDEX, "w") as index_file:
+    with INDEX.open("w", encoding="utf-8") as index_file:
         index_file.write(initial_text)
 
     examples_dir = os.path.join(ROOT, "examples")
@@ -238,7 +238,7 @@ def _main():
                     if not _add_table_entry(example_path, "advanced", "advanced"):
                         _add_table_entry(example_path, "", "other")
 
-    with open(INDEX, "a") as index_file:
+    with INDEX.open("a", encoding="utf-8") as index_file:
         index_file.write(categories["quickstart"]["table"])
 
         index_file.write("\nAdvanced Examples\n-----------------\n")

dev/check_pr_title.py → dev/devtool/check_pr_title.py

@@ -14,21 +14,20 @@
 # ==============================================================================
 """Used to check a given PR title format."""
 
-import pathlib
 import re
 import sys
 import tomllib
+from pathlib import Path
+from typing import Any
 
-if __name__ == "__main__":
 
-    pr_title = sys.argv[1]
+def _load_config() -> dict[str, Any]:
+    config_path = Path(__file__).resolve().parent.parent / "changelog_config.toml"
+    with config_path.open("rb") as file:
+        return tomllib.load(file)
 
-    # Load the YAML configuration
-    with (pathlib.Path(__file__).parent.resolve() / "changelog_config.toml").open(
-        "rb"
-    ) as file:
-        config = tomllib.load(file)
 
+def _validate_title(pr_title: str, config: dict[str, Any]) -> tuple[bool, str]:
     # Extract types, project, and scope from the config
     types = "|".join(config["type"])
     projects = "|".join(config["project"]) + "|\\*"
@@ -39,15 +38,15 @@
     pattern_template = config["pattern_template"]
     pattern = pattern_template.format(types=types, projects=projects, scope=scope)
 
-    # Check for the pattern in the first argument given to the script
+    # Check title against pattern.
     match = re.search(pattern, pr_title)
 
     valid = True
     error = "it doesn't have the correct format"
 
     # This check is there to ignore dependabot PRs from title checks
     if pr_title.startswith("build"):
-        sys.exit(0)
+        return True, ""
     elif not match:
         valid = False
     else:
@@ -57,8 +56,19 @@
         elif match.group(2) == "*" and match.group(3) is None:
             valid = False
             error = "the <PR_PROJECT> cannot be '*' without using the ':skip' flag"
+    return valid, error
+
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        raise ValueError("Usage: python -m devtool.check_pr_title '<title>'")
+
+    pr_title = sys.argv[1]
+    config = _load_config()
+    valid, error = _validate_title(pr_title, config)
 
     if not valid:
+        types = "|".join(config["type"])
         print(
             f"PR title `{pr_title}` is invalid, {error}.\n\nA PR title should "
             "be of the form:\n\n\t<PR_TYPE>(<PR_PROJECT>): <PR_SUBJECT>\n\n"
@@ -73,3 +83,7 @@
             "the changelog:\n\n\t`feat(framework:skip): Add new option to build CLI`\n"
         )
         sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

dev/update-html-themes.py → dev/devtool/update_html_themes.py

@@ -19,10 +19,11 @@
 import re
 from pathlib import Path
 from typing import Optional, Union
+
 import yaml
 
-# Change this if you want to search a different directory.
-ROOT_DIR = Path(".")
+REPO_ROOT = Path(__file__).resolve().parents[2]
+ROOT_DIR = REPO_ROOT
 
 # Define new fields to be added to the `html_theme_options` dictionary in `conf.py`.
 # If no fields are needed, set to an empty dictionary.
@@ -37,7 +38,7 @@
     },
 }
 
-with (ROOT_DIR / "dev" / "docs-ui-config.yml").open() as f:
+with (REPO_ROOT / "dev" / "docs-ui-config.yml").open(encoding="utf-8") as f:
     announcement = yaml.safe_load(f)["announcement"]
     if announcement["enabled"]:
         NEW_FIELDS["announcement"] = announcement["html"]

dev/update_python.py → dev/devtool/update_python.py

@@ -125,7 +125,7 @@ def _update_python_versions(
                     r"\g<1>" + new_major_minor + r"\g<2>",
                 ),
             ],
-            "dev/*.py": [
+            "dev/devtool/*.py": [
                 # Update version assignments
                 (
                     r'(["\'])' + re.escape(old_version) + r'(\.\d+)?(["\'],?)\s*\n?',

framework/docs/source/contributor-how-to-install-development-versions.rst

@@ -10,8 +10,8 @@ Using Poetry (recommended)
 ==========================
 
 Install a ``flwr`` pre-release from PyPI: update the ``flwr`` dependency in
-``pyproject.toml`` and then reinstall (don't forget to delete ``poetry.lock`` (``rm
-poetry.lock``) before running ``poetry install``).
+``pyproject.toml`` and then reinstall (ensure to delete ``poetry.lock`` via ``rm
+poetry.lock`` before running ``python -m poetry install``).
 
 - ``flwr = { version = "1.0.0a0", allow-prereleases = true }`` (without extras)
 - ``flwr = { version = "1.0.0a0", allow-prereleases = true, extras = ["simulation"] }``

framework/docs/source/contributor-how-to-set-up-a-virtual-env.rst

@@ -52,7 +52,8 @@ Activate the virtualenv by running the following command:
 ************************
 
 The Flower examples are based on `Poetry <https://python-poetry.org/docs/>`_ to manage
-dependencies. After installing Poetry you simply create a virtual environment with:
+dependencies. After installing Poetry you can create and activate a virtual environment
+with:
 
 .. code-block:: shell
 

framework/docs/source/contributor-how-to-write-documentation.rst

@@ -9,16 +9,16 @@
 The Flower documentation lives in the ``doc`` directory. The Sphinx-based documentation
 system supports both reStructuredText (``.rst`` files) and Markdown (``.md`` files).
 
-Note that, in order to build the documentation locally (with ``poetry run make html``,
-like described below), `Pandoc <https://pandoc.org/installing.html>`_ needs to be
-installed on the system.
+Note that, in order to build the documentation locally (with ``python -m poetry run make
+html``, as described below), `Pandoc <https://pandoc.org/installing.html>`_ needs to
+be installed on the system.
 
 ***********************
  Edit an existing page
 ***********************
 
 1. Edit an existing ``.rst`` (or ``.md``) file under ``framework/docs/source/``
-2. Compile the docs: ``cd framework/docs``, then ``poetry run make html``
+2. Compile the docs: ``cd framework/docs``, then ``python -m poetry run make html``
 3. Open ``framework/docs/build/html/index.html`` in the browser to check the result
 
 *******************
@@ -28,5 +28,5 @@ installed on the system.
 1. Add new ``.rst`` file under ``framework/docs/source/``
 2. Add content to the new ``.rst`` file
 3. Link to the new rst from ``index.rst``
-4. Compile the docs: ``cd framework/docs``, then ``poetry run make html``
+4. Compile the docs: ``cd framework/docs``, then ``python -m poetry run make html``
 5. Open ``framework/docs/build/html/index.html`` in the browser to check the result

framework/docs/source/how-to-upgrade-to-flower-1.0.rst

@@ -31,9 +31,9 @@ Here's how to update an existing installation to Flower 1.0 using either pip or
   - ``python -m pip install -U flwr`` (when using ``start_server`` and ``start_client``)
   - ``python -m pip install -U 'flwr[simulation]'`` (when using ``start_simulation``)
 
-- Poetry: update the ``flwr`` dependency in ``pyproject.toml`` and then reinstall (don't
-  forget to delete ``poetry.lock`` via ``rm poetry.lock`` before running ``poetry
-  install``).
+- Poetry: update the ``flwr`` dependency in ``pyproject.toml`` and then reinstall
+  (ensure to delete ``poetry.lock`` via ``rm poetry.lock`` before running ``python -m
+  poetry install``).
 
   - ``flwr = "^1.0.0"`` (when using ``start_server`` and ``start_client``)
   - ``flwr = { version = "^1.0.0", extras = ["simulation"] }`` (when using