Cherry-pick [2.7] Pin pandas<3.0 and fix pandas 3.x compatibility in federated statistics (#4227) (#4292)

YuanTingHsieh · chesterxgchen · claude · web-flow · commit 60c917a5724f · 2026-03-12T03:55:16.000+08:00
## Summary Starting from pandas 3.0, two breaking changes caused **all federated statistics jobs to fail** with `EXECUTION_RESULT_ERROR` on every client: **Issue 1: `AttributeError: 'StringDtype' object has no attribute 'char'`** pandas 3.0 enables `future.infer_string` by default, so `pd.read_csv()` infers string columns as `pd.StringDtype` — a pandas ExtensionDtype that has no `.char` attribute. This crashed `dtype_to_data_type()` in `numpy_utils.py` immediately on any DataFrame with string columns. **Issue 2: `AttributeError: 'Series' object has no attribute 'ravel'`** `Series.ravel()` was deprecated in pandas 2.2.0 and **removed** in pandas 3.0. Every `histogram()` method in the statistics module called `feature.ravel()` on a pandas Series. Both errors produced empty statistics output `{}` and affected `df_stats_job`, `df_stats_job_scale`, and `hierarchical_stats` jobs. ## Changes **Guard / Pin (prevents pandas 3.0 from being installed):** - Pin `pandas~=2.3` across 16 `requirements.txt` files (15 previously unpinned + new `df_stats/requirements.txt`) **Code fixes (forward-compatible with pandas 3.x when pin is eventually lifted):** - `nvflare/app_common/statistics/numpy_utils.py` — add `hasattr(dtype, "char")` guard in `dtype_to_data_type()`; ExtensionDtypes without `.char` (e.g. `StringDtype`) fall through to `DataType.STRING` - `nvflare/app_opt/statistics/df/df_core_statistics.py` — `feature.ravel()` → `feature.to_numpy()` - `examples/advanced/federated-statistics/hierarchical_stats/.../hierarchical_stats.py` — `feature.ravel()` → `feature.to_numpy()` - `tests/unit_test/app_common/executors/statistics/mock_df_stats_executor.py` — `feature.ravel()` → `feature.to_numpy()` **New file:** - `examples/advanced/federated-statistics/df_stats/requirements.txt` — was missing; created from `image_stats/requirements.txt` minus image-specific deps (`monai`, `kaleido`, `kagglehub`) **New tests:** - `tests/unit_test/app_common/statistics/numpy_utils_test.py` — 11 tests covering all `dtype_to_data_type()` branches including the new `pd.StringDtype` case ## Test plan - [ ] All 11 new unit tests in `numpy_utils_test.py` pass - [ ] Existing `statistics_executor_test.py` tests continue to pass (histogram path exercises `feature.to_numpy()`) - [ ] Run `df_stats_job` and `hierarchical_stats` jobs end-to-end with pandas 2.3 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Fixes # . ### Description A few sentences describing the changes proposed in this pull request. ### Types of changes  - [x] Non-breaking change (fix or new feature that would not break existing functionality). - [ ] Breaking change (fix or new feature that would cause existing functionality to change). - [ ] New tests added to cover the changes. - [ ] Quick tests passed locally by running `./runtest.sh`. - [ ] In-line docstrings updated. - [ ] Documentation updated. Co-authored-by: Chester Chen <512707+chesterxgchen@users.noreply.github.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
diff --git a/examples/advanced/cifar10/pt/cifar10-sim/requirements.txt b/examples/advanced/cifar10/pt/cifar10-sim/requirements.txt
@@ -4,5 +4,5 @@ torchvision
 tensorboard
 matplotlib
 seaborn
-pandas
+pandas~=2.3
 tbparse
diff --git a/examples/advanced/federated-statistics/df_stats/requirements.txt b/examples/advanced/federated-statistics/df_stats/requirements.txt
@@ -0,0 +1,6 @@
+nvflare~=2.7.2rc
+numpy
+pandas~=2.3
+matplotlib
+jupyter
+notebook
diff --git a/examples/advanced/federated-statistics/hierarchical_stats/jobs/hierarchical_stats/app/custom/hierarchical_stats.py b/examples/advanced/federated-statistics/hierarchical_stats/jobs/hierarchical_stats/app/custom/hierarchical_stats.py
@@ -105,7 +105,7 @@ def histogram(
 
         df = self.data[dataset_name]
         feature: Series = df[feature_name]
-        flattened = feature.ravel()
+        flattened = feature.to_numpy()
         flattened = flattened[flattened != np.array(None)]
         buckets = get_std_histogram_buckets(flattened, num_of_bins, BinRange(global_min_value, global_max_value))
         return Histogram(HistogramType.STANDARD, buckets)
diff --git a/examples/advanced/federated-statistics/hierarchical_stats/requirements.txt b/examples/advanced/federated-statistics/hierarchical_stats/requirements.txt
@@ -1,7 +1,7 @@
 nvflare~=2.7.0rc
 ipywidgets
 numpy
-pandas
+pandas~=2.3
 matplotlib
 jupyterlab
 jupyter
diff --git a/examples/advanced/federated-statistics/image_stats/requirements.txt b/examples/advanced/federated-statistics/image_stats/requirements.txt
@@ -1,7 +1,7 @@
 nvflare~=2.7.2rc
 numpy
 monai[itk]
-pandas
+pandas~=2.3
 kaleido
 matplotlib
 jupyter
diff --git a/examples/advanced/gnn/requirements.txt b/examples/advanced/gnn/requirements.txt
@@ -5,4 +5,4 @@ tensorboard
 scikit-learn
 tqdm
 filelock
-pandas
+pandas~=2.3
diff --git a/examples/advanced/job_api/sklearn/requirements.txt b/examples/advanced/job_api/sklearn/requirements.txt
@@ -1,4 +1,4 @@
 nvflare~=2.7.2rc
-pandas
+pandas~=2.3
 scikit-learn
 joblib
diff --git a/examples/advanced/monai/spleen_ct_segmentation/requirements.txt b/examples/advanced/monai/spleen_ct_segmentation/requirements.txt
@@ -6,7 +6,7 @@ monai>=1.3.0
 nvflare[PT]>=2.7.2rc
 tensorboard
 scikit-image
-pandas
+pandas~=2.3
 matplotlib
 monai[ignite,tqdm]
 huggingface_hub
diff --git a/examples/advanced/psi/user_email_match/requirements.txt b/examples/advanced/psi/user_email_match/requirements.txt
@@ -1,3 +1,3 @@
 nvflare~=2.7.2rc
 openmined-psi>=2.0.5
-pandas
+pandas~=2.3
diff --git a/examples/advanced/sklearn-kmeans/requirements.txt b/examples/advanced/sklearn-kmeans/requirements.txt
@@ -1,5 +1,5 @@
 nvflare~=2.7.2rc
-pandas
+pandas~=2.3
 scikit-learn
 joblib
 tensorboard
diff --git a/examples/advanced/sklearn-linear/requirements.txt b/examples/advanced/sklearn-linear/requirements.txt
@@ -1,5 +1,5 @@
 nvflare~=2.7.2rc
-pandas
+pandas~=2.3
 scikit-learn
 joblib
 tensorboard
diff --git a/examples/advanced/sklearn-svm/requirements.txt b/examples/advanced/sklearn-svm/requirements.txt
@@ -1,5 +1,5 @@
 nvflare~=2.7.2rc
-pandas
+pandas~=2.3
 scikit-learn
 joblib
 tensorboard
diff --git a/examples/advanced/vertical_federated_learning/cifar10-splitnn/requirements.txt b/examples/advanced/vertical_federated_learning/cifar10-splitnn/requirements.txt
@@ -2,4 +2,4 @@ torch
 torchvision
 tensorboard
 openmined-psi>=2.0.5
-pandas
+pandas~=2.3
diff --git a/examples/advanced/xgboost/requirements.txt b/examples/advanced/xgboost/requirements.txt
@@ -1,7 +1,7 @@
 nvflare~=2.7.2
 kagglehub
 openmined-psi>=2.0.5
-pandas
+pandas~=2.3
 torch
 scikit-learn
 shap
diff --git a/examples/hello-world/hello-dp/requirements.txt b/examples/hello-world/hello-dp/requirements.txt
@@ -3,4 +3,4 @@ torch
 tensorboard
 scikit-learn
 opacus
-pandas
+pandas~=2.3
diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-7_algorithms_and_workflows/07.2_algorithms/07.2.4_split_learning/requirements.txt b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-7_algorithms_and_workflows/07.2_algorithms/07.2.4_split_learning/requirements.txt
@@ -3,4 +3,4 @@ torch
 torchvision
 tensorboard
 openmined-psi>=2.0.5
-pandas
+pandas~=2.3
diff --git a/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-8_federated_LLM_training/08.1_fed_bert/requirements.txt b/examples/tutorials/self-paced-training/part-4_advanced_federated_learning/chapter-8_federated_LLM_training/08.1_fed_bert/requirements.txt
@@ -3,5 +3,5 @@ torch>=2.2.0
 torchvision
 tensorboard
 transformers>=4.40.0
-pandas
+pandas~=2.3
 seqeval
diff --git a/examples/tutorials/self-paced-training/part-5_federated_learning_applications_in_industries/chapter-11_federated_learning_in_healthcare_lifescience/11.3_medical_devices/code/docker/requirements.txt b/examples/tutorials/self-paced-training/part-5_federated_learning_applications_in_industries/chapter-11_federated_learning_in_healthcare_lifescience/11.3_medical_devices/code/docker/requirements.txt
@@ -1,3 +1,3 @@
 nvflare
-pandas
+pandas~=2.3
 numpy
diff --git a/nvflare/app_common/statistics/numpy_utils.py b/nvflare/app_common/statistics/numpy_utils.py
@@ -16,6 +16,7 @@
 from typing import List, Optional
 
 import numpy as np
+from pandas.api.types import is_bool_dtype, is_datetime64_any_dtype, is_float_dtype, is_integer_dtype
 
 from nvflare.app_common.abstract.statistics_spec import Bin, BinRange, DataType
 
@@ -32,11 +33,19 @@ def default(self, obj):
 
 
 def dtype_to_data_type(dtype) -> DataType:
-    if dtype.char in np.typecodes["AllFloat"]:
+    # Use pandas type-checking functions so that both numpy dtypes and pandas
+    # nullable ExtensionDtypes (Int64Dtype, Float64Dtype, BooleanDtype, StringDtype, …)
+    # are classified correctly.
+    if is_float_dtype(dtype):
         return DataType.FLOAT
-    elif dtype.char in np.typecodes["AllInteger"] or dtype == bool:
+    elif is_bool_dtype(dtype):
+        # is_bool must be checked before is_integer because BooleanDtype satisfies both
         return DataType.INT
-    elif np.issubdtype(dtype, np.datetime64) or np.issubdtype(dtype, np.timedelta64):
+    elif is_integer_dtype(dtype):
+        return DataType.INT
+    elif is_datetime64_any_dtype(dtype) or (
+        hasattr(dtype, "kind") and dtype.kind == "m"  # np.timedelta64 has kind "m"
+    ):
         return DataType.DATETIME
     else:
         return DataType.STRING
diff --git a/nvflare/app_opt/statistics/df/df_core_statistics.py b/nvflare/app_opt/statistics/df/df_core_statistics.py
@@ -77,7 +77,7 @@ def histogram(
 
         df = self.data[dataset_name]
         feature: Series = df[feature_name]
-        flattened = feature.ravel()
+        flattened = feature.to_numpy()
         flattened = flattened[flattened != np.array(None)]
         buckets = get_std_histogram_buckets(flattened, num_of_bins, BinRange(global_min_value, global_max_value))
         return Histogram(HistogramType.STANDARD, buckets)
diff --git a/tests/unit_test/app_common/executors/statistics/mock_df_stats_executor.py b/tests/unit_test/app_common/executors/statistics/mock_df_stats_executor.py
@@ -88,7 +88,7 @@ def histogram(
 
         df = self.data[dataset_name]
         feature: Series = df[feature_name]
-        flattened = feature.ravel()
+        flattened = feature.to_numpy()
         flattened = flattened[flattened != np.array(None)]
         buckets = get_std_histogram_buckets(flattened, num_of_bins, BinRange(global_min_value, global_max_value))
         return Histogram(HistogramType.STANDARD, buckets)
diff --git a/tests/unit_test/app_common/statistics/numpy_utils_test.py b/tests/unit_test/app_common/statistics/numpy_utils_test.py
@@ -0,0 +1,63 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from nvflare.app_common.abstract.statistics_spec import DataType
+from nvflare.app_common.statistics.numpy_utils import dtype_to_data_type
+
+
+class TestDtypeToDataType:
+    @pytest.mark.parametrize("dtype", [np.dtype("float32"), np.dtype("float64")])
+    def test_float_dtypes(self, dtype):
+        assert dtype_to_data_type(dtype) == DataType.FLOAT
+
+    @pytest.mark.parametrize("dtype", [np.dtype("int32"), np.dtype("int64"), np.dtype("uint8")])
+    def test_int_dtypes(self, dtype):
+        assert dtype_to_data_type(dtype) == DataType.INT
+
+    def test_bool_dtype(self):
+        assert dtype_to_data_type(np.dtype("bool")) == DataType.INT
+
+    def test_datetime_dtype(self):
+        assert dtype_to_data_type(np.dtype("datetime64[ns]")) == DataType.DATETIME
+
+    def test_timedelta_dtype(self):
+        assert dtype_to_data_type(np.dtype("timedelta64[ns]")) == DataType.DATETIME
+
+    def test_object_dtype(self):
+        assert dtype_to_data_type(np.dtype("object")) == DataType.STRING
+
+    def test_pandas_string_dtype(self):
+        # pd.StringDtype has no .char attribute — this is the pandas 3.0 breaking case
+        assert dtype_to_data_type(pd.StringDtype()) == DataType.STRING
+
+    def test_pandas_string_dtype_inferred_from_series(self):
+        # Simulate pandas 3.0 behavior where read_csv infers string columns as StringDtype
+        s = pd.array(["a", "b", "c"], dtype=pd.StringDtype())
+        assert dtype_to_data_type(s.dtype) == DataType.STRING
+
+    def test_pandas_nullable_int_dtype(self):
+        # pd.Int64Dtype is a nullable ExtensionDtype — must map to INT, not STRING
+        assert dtype_to_data_type(pd.Int64Dtype()) == DataType.INT
+
+    def test_pandas_nullable_float_dtype(self):
+        # pd.Float64Dtype is a nullable ExtensionDtype — must map to FLOAT, not STRING
+        assert dtype_to_data_type(pd.Float64Dtype()) == DataType.FLOAT
+
+    def test_pandas_nullable_bool_dtype(self):
+        # pd.BooleanDtype is a nullable ExtensionDtype — must map to INT, not STRING
+        assert dtype_to_data_type(pd.BooleanDtype()) == DataType.INT
