Require PyArrow >=16 (#9185)

Author: crusaderky

.github/workflows/tests.yaml

@@ -75,12 +75,12 @@ jobs:
           - os: ubuntu-latest
             environment: mindeps
             label: pandas
-            extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1]
+            extra_packages: [numpy=1.24, pandas=2.0, pyarrow=16.0]
             partition: "ci1"
           - os: ubuntu-latest
             environment: mindeps
             label: pandas
-            extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1]
+            extra_packages: [numpy=1.24, pandas=2.0, pyarrow=16.0]
             partition: "not ci1"
 
           - os: ubuntu-latest

continuous_integration/environment-3.11.yaml

@@ -28,7 +28,7 @@ dependencies:
   - pre-commit
   - prometheus_client
   - psutil
-  - pyarrow=14
+  - pyarrow=16.0
   - pytest
   - pytest-cov
   - pytest-faulthandler

continuous_integration/recipes/dask/meta.yaml

@@ -27,11 +27,11 @@ requirements:
     - {{ pin_compatible('distributed', exact=True) }}
     - cytoolz >=0.11.2
     - lz4 >=4.3.2
-    - numpy >=1.21
+    - numpy >=1.24
     - pandas >=2
     - bokeh >=3.1.0
     - jinja2 >=2.10.3
-    - pyarrow >=7.0
+    - pyarrow >=16
 
   run_constrained:
     - openssl !=1.1.1e

distributed/protocol/arrow.py

@@ -4,10 +4,9 @@
 
 from distributed.protocol.serialize import dask_deserialize, dask_serialize
 
-if pyarrow.__version__ < "0.10":
+if int(pyarrow.__version__.split(".")[0]) < 16:
     raise ImportError(
-        "Need pyarrow >= 0.10 . "
-        "See https://arrow.apache.org/docs/python/install.html"
+        "Need pyarrow >=16.0. See https://arrow.apache.org/docs/python/install.html"
     )
 
 

distributed/shuffle/tests/test_shuffle.py

@@ -1084,7 +1084,6 @@ async def test_heartbeat(c, s, a, b):
 
 
 @pytest.mark.skipif("not pa", reason="Requires PyArrow")
-@pytest.mark.filterwarnings("ignore:DatetimeTZBlock")  # pandas >=2.2 vs. pyarrow <15
 @pytest.mark.parametrize("drop_column", [True, False])
 def test_processing_chain(tmp_path, drop_column):
     """

pyproject.toml

@@ -178,8 +178,6 @@ filterwarnings = [
     '''ignore:datetime\.datetime\.utc(fromtimestamp|now)\(\) is deprecated and scheduled for removal in a future version.*:DeprecationWarning:tornado''',
     # https://github.com/dateutil/dateutil/issues/1284
     '''ignore:datetime\.datetime\.utc(fromtimestamp|now)\(\) is deprecated and scheduled for removal in a future version.*:DeprecationWarning:dateutil''',
-    # https://github.com/dask/dask/pull/10622
-    '''ignore:Minimal version of pyarrow will soon be increased to 14.0.1''',
     '''ignore:the matrix subclass is not the recommended way''',
     '''ignore:The current Dask DataFrame implementation is deprecated.*:DeprecationWarning''',
 ]