Merge branch 'main' into an-add-docs

Author: nadzhou

.github/workflows/test-report.yaml

@@ -35,7 +35,7 @@ jobs:
       - name: mamba list
         run: mamba list
 
-      - uses: actions/cache@v4
+      - uses: actions/cache@v5
         id: cache
         with:
           # Suffix is depending on the backend / OS. Let's be agnostic here
@@ -53,7 +53,7 @@ jobs:
           mv test_report.html test_short_report.html deploy/
 
       - name: Deploy 🚀
-        uses: JamesIves/github-pages-deploy-action@v4.7.4
+        uses: JamesIves/github-pages-deploy-action@v4.8.0
         with:
           branch: gh-pages
           folder: deploy

.github/workflows/tests.yaml

@@ -26,20 +26,22 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        environment: [mindeps, "3.10", "3.11", "3.12", "3.13"]
+        environment: [mindeps, "3.10", "3.11", "3.12", "3.13", "3.14"]
         label: [default]
         extra_packages: [null]
         # Cherry-pick test modules to split the overall runtime roughly in half
         partition: [ci1, not ci1]
 
         exclude:
-          # MacOS CI does not have any hosts available; run it on 3.12 only
+          # MacOS CI does not have many hosts available; run it on 3.14 only
           - os: macos-latest
             environment: mindeps
           - os: macos-latest
             environment: "3.10"
           - os: macos-latest
             environment: "3.11"
+          - os: macos-latest
+            environment: "3.12"
           - os: macos-latest
             environment: "3.13"
 
@@ -73,12 +75,12 @@ jobs:
           - os: ubuntu-latest
             environment: mindeps
             label: pandas
-            extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1]
+            extra_packages: [numpy=1.24, pandas=2.0, pyarrow=16.0]
             partition: "ci1"
           - os: ubuntu-latest
             environment: mindeps
             label: pandas
-            extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1]
+            extra_packages: [numpy=1.24, pandas=2.0, pyarrow=16.0]
             partition: "not ci1"
 
           - os: ubuntu-latest
@@ -145,7 +147,7 @@ jobs:
 
       - name: Cache Conda env
         if: steps.skip-caching.outputs.trigger-found != 'true'
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ${{ env.CONDA }}/envs
           key: conda-${{ matrix.os }}-${{ matrix.environment }}-${{ matrix.label }}-${{ steps.get-date.outputs.today }}-${{ hashFiles(env.CONDA_FILE) }}-${{ env.CACHE_NUMBER }}
@@ -281,7 +283,7 @@ jobs:
         if: >
           always() &&
           (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure')
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: ${{ env.TEST_ID }}
           path: reports
@@ -291,7 +293,7 @@ jobs:
         if: >
           always() &&
           (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure')
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: ${{ env.TEST_ID }}_cluster_dumps
           path: test_cluster_dump
@@ -305,7 +307,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Upload
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: Event File
           path: ${{ github.event_path }}

.pre-commit-config.yaml

@@ -1,22 +1,22 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.5
+    rev: v0.14.10
     hooks:
       - id: ruff-check
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 25.11.0
+    rev: 25.12.0
     hooks:
       - id: black
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.1
     hooks:
       -   id: codespell
           additional_dependencies:
             - tomli
           types_or: [rst, markdown]
           files: docs
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.2
+    rev: v1.19.1
     hooks:
       - id: mypy
         # Override default --ignore-missing-imports

continuous_integration/environment-3.10.yaml

@@ -20,7 +20,7 @@ dependencies:
   - ipykernel
   - ipywidgets
   - jinja2
-  - jupyter_events<0.11
+  - jupyter_events
   - jupyter-server-proxy
   - jupyterlab
   - locket
@@ -33,7 +33,7 @@ dependencies:
   - psutil
   - pyarrow
   - nvidia-ml-py  # Only tested here
-  - pytest<8.4  # Pin due to https://github.com/pytest-dev/pytest-cov/issues/693
+  - pytest
   - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
@@ -45,16 +45,12 @@ dependencies:
   - scikit-learn
   - scipy
   - sortedcollections
-  - tblib !=3.2.0,!=3.2.1
+  - tblib
   - toolz
   - torchvision  # Only tested here
   - tornado
   - zict  # overridden by git tip below
   - zstandard
-  # Temporary fix for https://github.com/pypa/setuptools/issues/4496
-  - setuptools < 71
-  # Temporary fix for https://github.com/jupyterlab/jupyterlab/issues/17012
-  - httpx<0.28.0
   - pip:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask/zict

continuous_integration/environment-3.11.yaml

@@ -18,7 +18,7 @@ dependencies:
   - ipykernel
   - ipywidgets
   - jinja2
-  - jupyter_events<0.11
+  - jupyter_events
   - jupyter-server-proxy
   - jupyterlab
   - locket
@@ -28,8 +28,8 @@ dependencies:
   - pre-commit
   - prometheus_client
   - psutil
-  - pyarrow=14
-  - pytest<8.4  # Pin due to https://github.com/pytest-dev/pytest-cov/issues/693
+  - pyarrow=16.0
+  - pytest
   - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
@@ -39,15 +39,11 @@ dependencies:
   - scikit-learn
   - scipy
   - sortedcollections
-  - tblib !=3.2.0,!=3.2.1
+  - tblib
   - toolz
   - tornado
   - zict  # overridden by git tip below
   - zstandard
-  # Temporary fix for https://github.com/pypa/setuptools/issues/4496
-  - setuptools < 71
-  # Temporary fix for https://github.com/jupyterlab/jupyterlab/issues/17012
-  - httpx<0.28.0
   - pip:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask/zict

continuous_integration/environment-3.12.yaml

@@ -18,7 +18,7 @@ dependencies:
   - ipykernel
   - ipywidgets
   - jinja2
-  - jupyter_events<0.11
+  - jupyter_events
   - jupyter-server-proxy
   - jupyterlab
   - locket
@@ -29,7 +29,7 @@ dependencies:
   - prometheus_client
   - psutil
   - pyarrow
-  - pytest<8.4  # Pin due to https://github.com/pytest-dev/pytest-cov/issues/693
+  - pytest
   - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
@@ -39,15 +39,11 @@ dependencies:
   - scikit-learn
   - scipy
   - sortedcollections
-  - tblib !=3.2.0,!=3.2.1
+  - tblib
   - toolz
   - tornado
   - zict  # overridden by git tip below
   - zstandard
-  # Temporary fix for https://github.com/pypa/setuptools/issues/4496
-  - setuptools < 71
-  # Temporary fix for https://github.com/jupyterlab/jupyterlab/issues/17012
-  - httpx<0.28.0
   - pip:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask/zict

continuous_integration/environment-3.13.yaml

@@ -17,7 +17,7 @@ dependencies:
   - ipykernel
   - ipywidgets
   - jinja2
-  - jupyter_events<0.11
+  - jupyter_events
   - jupyter-server-proxy
   - jupyterlab
   - locket
@@ -28,7 +28,7 @@ dependencies:
   - prometheus_client
   - psutil
   - pyarrow
-  - pytest<8.4  # Pin due to https://github.com/pytest-dev/pytest-cov/issues/693
+  - pytest
   - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
@@ -38,15 +38,11 @@ dependencies:
   - scikit-learn
   - scipy
   - sortedcollections
-  - tblib !=3.2.0,!=3.2.1
+  - tblib
   - toolz
   - tornado
   - zict  # overridden by git tip below
   - zstandard
-  # Temporary fix for https://github.com/pypa/setuptools/issues/4496
-  - setuptools < 71
-  # Temporary fix for https://github.com/jupyterlab/jupyterlab/issues/17012
-  - httpx<0.28.0
   - pip:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask/zict

continuous_integration/environment-3.14.yaml

@@ -0,0 +1,50 @@
+name: dask-distributed-313
+channels:
+  - conda-forge
+dependencies:
+  - python=3.14
+  - packaging
+  - pip
+  - asyncssh
+  - bokeh>3
+  - click
+  - cloudpickle
+  - coverage
+  - dask  # overridden by git tip below
+  - fsspec  # overridden by git tip below
+  - gilknocker
+  - h5py
+  - ipykernel
+  - ipywidgets
+  - jinja2
+  - jupyter_events
+  - jupyter-server-proxy
+  - jupyterlab
+  - locket
+  - msgpack-python
+  - netcdf4
+  - paramiko
+  - pre-commit
+  - prometheus_client
+  - psutil
+  - pyarrow
+  - pytest
+  - pytest-cov
+  - pytest-faulthandler
+  - pytest-repeat
+  - pytest-rerunfailures
+  - pytest-timeout
+  - requests
+  - scikit-learn
+  - scipy
+  - sortedcollections
+  - tblib
+  - toolz
+  - tornado
+  - zict  # overridden by git tip below
+  - zstandard
+  - pip:
+      - git+https://github.com/dask/dask
+      - git+https://github.com/dask/zict
+      - git+https://github.com/fsspec/filesystem_spec
+      - keras

continuous_integration/environment-mindeps.yaml

@@ -10,6 +10,9 @@ dependencies:
   - locket=1.0.0
   - msgpack-python=1.0.2
   - packaging=20.0
+  - pyparsing<3.3  # Newer versions of pyparsing break with old versions of packaging
+  - setuptools<71  # Newer versions of setuptools break with packaging<24
+                   # https://github.com/pypa/setuptools/issues/4496
   - psutil=5.8.0
   - pyyaml=5.4.1
   - sortedcontainers=2.0.5
@@ -18,8 +21,6 @@ dependencies:
   - tornado=6.2.0
   - urllib3=1.26.5
   - zict=3.0.0
-  # Temporary fix for https://github.com/pypa/setuptools/issues/4496
-  - setuptools < 71
   # Distributed depends on the latest version of Dask
   - pip
   - pip:

continuous_integration/recipes/dask/meta.yaml

@@ -27,11 +27,11 @@ requirements:
     - {{ pin_compatible('distributed', exact=True) }}
     - cytoolz >=0.11.2
     - lz4 >=4.3.2
-    - numpy >=1.21
+    - numpy >=1.24
     - pandas >=2
     - bokeh >=3.1.0
     - jinja2 >=2.10.3
-    - pyarrow >=7.0
+    - pyarrow >=16
 
   run_constrained:
     - openssl !=1.1.1e