[Data] Cap pandas to <3 for ray[data] and make SettingWithCopyWarning handling pandas-3-safe

python/ray/air/util/data_batch_conversion.py

@@ -286,11 +286,16 @@ def _cast_ndarray_columns_to_tensor_extension(df: "pd.DataFrame") -> "pd.DataFra
     Cast all NumPy ndarray columns in df to our tensor extension type, TensorArray.
     """
     pd = _lazy_import_pandas()
+    # Pandas has moved/renamed this warning across versions, and pandas 3.x may not
+    # expose it at all. If we can't find it, just don't attempt to filter it.
+    SettingWithCopyWarning = None
     try:
         SettingWithCopyWarning = pd.core.common.SettingWithCopyWarning
-    except AttributeError:
-        # SettingWithCopyWarning was moved to pd.errors in Pandas 1.5.0.
-        SettingWithCopyWarning = pd.errors.SettingWithCopyWarning
+    except Exception:
+        try:
+            SettingWithCopyWarning = pd.errors.SettingWithCopyWarning
+        except Exception:
+            SettingWithCopyWarning = None
 
     from ray.data._internal.tensor_extensions.pandas import (
         TensorArray,
@@ -313,7 +318,10 @@ def _cast_ndarray_columns_to_tensor_extension(df: "pd.DataFrame") -> "pd.DataFra
                 # https://stackoverflow.com/a/74193599
                 with warnings.catch_warnings():
                     warnings.simplefilter("ignore", category=FutureWarning)
-                    warnings.simplefilter("ignore", category=SettingWithCopyWarning)
+                    if SettingWithCopyWarning is not None:
+                        warnings.simplefilter(
+                            "ignore", category=SettingWithCopyWarning
+                        )
                     df[col_name] = TensorArray(col)
             except Exception as e:
                 raise ValueError(
@@ -329,11 +337,14 @@ def _cast_ndarray_columns_to_tensor_extension(df: "pd.DataFrame") -> "pd.DataFra
 def _cast_tensor_columns_to_ndarrays(df: "pd.DataFrame") -> "pd.DataFrame":
     """Cast all tensor extension columns in df to NumPy ndarrays."""
     pd = _lazy_import_pandas()
+    SettingWithCopyWarning = None
     try:
         SettingWithCopyWarning = pd.core.common.SettingWithCopyWarning
-    except AttributeError:
-        # SettingWithCopyWarning was moved to pd.errors in Pandas 1.5.0.
-        SettingWithCopyWarning = pd.errors.SettingWithCopyWarning
+    except Exception:
+        try:
+            SettingWithCopyWarning = pd.errors.SettingWithCopyWarning
+        except Exception:
+            SettingWithCopyWarning = None
     from ray.data._internal.tensor_extensions.pandas import TensorDtype
 
     # Try to convert any tensor extension columns to ndarray columns.
@@ -348,6 +359,7 @@ def _cast_tensor_columns_to_ndarrays(df: "pd.DataFrame") -> "pd.DataFrame":
             # https://stackoverflow.com/a/74193599
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", category=FutureWarning)
-                warnings.simplefilter("ignore", category=SettingWithCopyWarning)
+                if SettingWithCopyWarning is not None:
+                    warnings.simplefilter("ignore", category=SettingWithCopyWarning)
                 df[col_name] = list(col.to_numpy())
     return df

python/ray/data/util/data_batch_conversion.py

@@ -221,11 +221,16 @@ def _cast_ndarray_columns_to_tensor_extension(df: "pd.DataFrame") -> "pd.DataFra
     Cast all NumPy ndarray columns in df to our tensor extension type, TensorArray.
     """
     pd = _lazy_import_pandas()
+    # Pandas has moved/renamed this warning across versions, and pandas 3.x may not
+    # expose it at all. If we can't find it, just don't attempt to filter it.
+    SettingWithCopyWarning = None
     try:
         SettingWithCopyWarning = pd.core.common.SettingWithCopyWarning
-    except AttributeError:
-        # SettingWithCopyWarning was moved to pd.errors in Pandas 1.5.0.
-        SettingWithCopyWarning = pd.errors.SettingWithCopyWarning
+    except Exception:
+        try:
+            SettingWithCopyWarning = pd.errors.SettingWithCopyWarning
+        except Exception:
+            SettingWithCopyWarning = None
 
     from ray.data._internal.tensor_extensions.pandas import (
         TensorArray,
@@ -248,7 +253,10 @@ def _cast_ndarray_columns_to_tensor_extension(df: "pd.DataFrame") -> "pd.DataFra
                 # https://stackoverflow.com/a/74193599
                 with warnings.catch_warnings():
                     warnings.simplefilter("ignore", category=FutureWarning)
-                    warnings.simplefilter("ignore", category=SettingWithCopyWarning)
+                    if SettingWithCopyWarning is not None:
+                        warnings.simplefilter(
+                            "ignore", category=SettingWithCopyWarning
+                        )
                     df[col_name] = TensorArray(col)
             except Exception as e:
                 raise ValueError(
@@ -264,11 +272,14 @@ def _cast_ndarray_columns_to_tensor_extension(df: "pd.DataFrame") -> "pd.DataFra
 def _cast_tensor_columns_to_ndarrays(df: "pd.DataFrame") -> "pd.DataFrame":
     """Cast all tensor extension columns in df to NumPy ndarrays."""
     pd = _lazy_import_pandas()
+    SettingWithCopyWarning = None
     try:
         SettingWithCopyWarning = pd.core.common.SettingWithCopyWarning
-    except AttributeError:
-        # SettingWithCopyWarning was moved to pd.errors in Pandas 1.5.0.
-        SettingWithCopyWarning = pd.errors.SettingWithCopyWarning
+    except Exception:
+        try:
+            SettingWithCopyWarning = pd.errors.SettingWithCopyWarning
+        except Exception:
+            SettingWithCopyWarning = None
     from ray.data._internal.tensor_extensions.pandas import TensorDtype
 
     # Try to convert any tensor extension columns to ndarray columns.
@@ -283,6 +294,7 @@ def _cast_tensor_columns_to_ndarrays(df: "pd.DataFrame") -> "pd.DataFrame":
             # https://stackoverflow.com/a/74193599
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", category=FutureWarning)
-                warnings.simplefilter("ignore", category=SettingWithCopyWarning)
+                if SettingWithCopyWarning is not None:
+                    warnings.simplefilter("ignore", category=SettingWithCopyWarning)
                 df[col_name] = list(col.to_numpy())
     return df

python/requirements.txt

@@ -46,13 +46,13 @@ aiohttp_cors
 dm_tree
 uvicorn
 prometheus_client>=0.7.1
-pandas
+pandas>=1.3,<3
 tensorboardX
 aiohttp>=3.13.3
 starlette
 typer
 fsspec
-pandas>=1.3
+pandas>=1.3,<3
 pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*,!=2.11.*,<3  # Serve users can use pydantic<2
 py-spy>=0.2.0; python_version < '3.12'
 py-spy>=0.4.0; python_version >= '3.12'

python/setup.py

@@ -218,7 +218,10 @@ def get_packages(self):
 # also update the matching section of requirements/requirements.txt
 # in this directory
 if setup_spec.type == SetupType.RAY:
-    pandas_dep = "pandas >= 1.3"
+    # Ray Data currently relies on pandas APIs that are not available in pandas 3.x
+    # (e.g. SettingWithCopyWarning). Keep an upper bound until full pandas 3 support
+    # is added.
+    pandas_dep = "pandas >= 1.3, < 3"
     numpy_dep = "numpy >= 1.20"
     pyarrow_deps = [
         "pyarrow >= 9.0.0",
@@ -270,7 +273,7 @@ def get_packages(self):
             "watchfiles",
         ],
         "tune": [
-            "pandas",
+            pandas_dep,
             # TODO: Remove pydantic dependency from tune once tune doesn't import train
             pydantic_dep,
             "tensorboardX>=1.9",