feat: add pandas 3.x compatibility and upgrade (#1491)

 - Bump pandas dependency to >=3,<4
 - Fix Copy-on-Write incompatible patterns (chained indexing, slice assignment)
 - Fix Index.values returning StringArray instead of ndarray

Author: kjbrak

.basedpyright/baseline.json

@@ -1070,14 +1070,6 @@
                     "endColumn": 43,
                     "lineCount": 1
                 }
-            },
-            {
-                "code": "reportReturnType",
-                "range": {
-                    "startColumn": 15,
-                    "endColumn": 64,
-                    "lineCount": 1
-                }
             }
         ],
         "./src/libecalc/domain/process/compressor/core/sampled/compressor_model_sampled_2d.py": [

pyproject.toml

@@ -31,7 +31,7 @@ dependencies = [
     "pydantic<3",
     "PyYAML==6.*",
     "numpy<3",
-    "pandas==2.*",
+    "pandas>=3,<4",
     "scipy<2",
     "ruamel-yaml~=0.18",
     "Shapely==2.*",

src/libecalc/domain/process/compressor/core/sampled/compressor_model_sampled.py

@@ -414,7 +414,7 @@ def _non_degenerated_variables(sampled_data: pd.DataFrame) -> list[str]:
 
         """
         uniques = sampled_data.apply(lambda x: x.nunique())
-        return array_to_list(uniques[uniques != 1].index.values)
+        return list(uniques[uniques != 1].index)
 
     @dataclass
     class Turbine:

src/libecalc/domain/process/compressor/core/sampled/compressor_model_sampled_2d.py

@@ -50,16 +50,16 @@ def __init__(self, sampled_data: pd.DataFrame, function_header: str):
         """
         # the interpolator we use when calculating energy usage
         self._interpolator = LinearNDInterpolator(
-            sampled_data[self.variable_order_2d][:].values,
-            sampled_data[function_header[:]][:].values,  # type: ignore[arg-type]
+            sampled_data[self.variable_order_2d].to_numpy(),
+            sampled_data[function_header].to_numpy(),  # type: ignore[arg-type]
             fill_value=np.nan,
             rescale=True,
         )
 
         # we define a convex hull for the working area for the compressor
         # for any point (rate, Dp) in the convex hull we can draw a line between
         # them to interpolate and find interpolated values
-        convex_hull = ConvexHull(sampled_data[self.variable_order_2d][:].values)
+        convex_hull = ConvexHull(sampled_data[self.variable_order_2d].to_numpy())
 
         # we can only increase rate, therefore we do not care about upper_rate_points...
         # we either want upper or lower convex hull, so for Pd we want lower
@@ -217,13 +217,13 @@ class CompressorModelSampled2DRatePs:
 
     def __init__(self, sampled_data: pd.DataFrame, function_header: str):
         self._interpolator = LinearNDInterpolator(
-            sampled_data[self.variable_order_2d][:].values,
-            sampled_data[function_header[:]][:].values,  # type: ignore[arg-type]
+            sampled_data[self.variable_order_2d].to_numpy(),
+            sampled_data[function_header].to_numpy(),  # type: ignore[arg-type]
             fill_value=np.nan,
             rescale=True,
         )
 
-        convex_hull = ConvexHull(sampled_data[self.variable_order_2d][:].values)
+        convex_hull = ConvexHull(sampled_data[self.variable_order_2d].to_numpy())
         (
             lower_rate_points_qh,
             _,
@@ -292,13 +292,13 @@ class CompressorModelSampled2DPsPd:
 
     def __init__(self, sampled_data: pd.DataFrame, function_header: str):
         self._interpolator = LinearNDInterpolator(
-            sampled_data[self.variable_order_2d][:].values,
-            sampled_data[function_header[:]][:].values,  # type: ignore[arg-type]
+            sampled_data[self.variable_order_2d].to_numpy(),
+            sampled_data[function_header].to_numpy(),  # type: ignore[arg-type]
             fill_value=np.nan,
             rescale=True,
         )
 
-        convex_hull = ConvexHull(sampled_data[self.variable_order_2d][:].values)
+        convex_hull = ConvexHull(sampled_data[self.variable_order_2d].to_numpy())
         lower_pd_points_qh, _, _ = get_lower_upper_qhull(convex_hull, axis=self.pd_axis, case_2d="ps_pd")
         lower_pd_points = lower_pd_points_qh.points
 

src/libecalc/domain/process/compressor/core/sampled/compressor_model_sampled_3d.py

@@ -124,17 +124,17 @@ def __init__(
             self._scale_factor_rate = round(
                 2 * sampled_data[RATE_NAME].mean() / (sampled_data[PS_NAME].mean() + sampled_data[PD_NAME].mean())
             )
-            sampled_data_scaled.loc[:, RATE_NAME] = sampled_data_scaled.loc[:, RATE_NAME] / self._scale_factor_rate
+            sampled_data_scaled[RATE_NAME] = sampled_data_scaled[RATE_NAME] / self._scale_factor_rate
         else:
             self._do_rescale = False
             self._scale_factor_rate = 1.0
 
-        qhull_points = sampled_data_scaled[self.variable_order_3d][:].values
+        qhull_points = sampled_data_scaled[self.variable_order_3d].to_numpy()
         convex_hull = ConvexHull(qhull_points)
         delaunay = Delaunay(qhull_points)
         interpolator = LinearNDInterpolator(
             points=delaunay,
-            values=sampled_data[function_header[:]][:].values,  # type: ignore[arg-type]
+            values=sampled_data[function_header].to_numpy(),  # type: ignore[arg-type]
             fill_value=np.nan,
             rescale=False,
         )

tests/libecalc/core/models/compressor_modelling/sampled/test_convex_hull_common.py

@@ -40,7 +40,7 @@ def test_get_lower_upper_qhull():
     # func_header = EcalcYamlKeywords.consumer_tabular_fuel
     df.columns = variables  # + [func_header]
 
-    qh = ConvexHull(df[variables][:].values)
+    qh = ConvexHull(df[variables].to_numpy())
 
     # Test in all three directions
     lower_p, upper_p, _ = get_lower_upper_qhull(qh, axis=0)
@@ -78,7 +78,7 @@ def test_get_lower_upper_qhull():
 
     df.columns = variables
 
-    qh = ConvexHull(df[variables][:].values)
+    qh = ConvexHull(df[variables].to_numpy())
 
     # axis=0
     lower_p, upper_p, _ = get_lower_upper_qhull(qh, axis=0)
@@ -121,7 +121,7 @@ def test_get_lower_upper_qhull():
     df = pd.DataFrame(df_data)
     df.columns = variables
 
-    qh = ConvexHull(df[variables][:].values)
+    qh = ConvexHull(df[variables].to_numpy())
 
     # axis=0
     lower_p, upper_p, _ = get_lower_upper_qhull(qh, axis=0)
@@ -314,7 +314,7 @@ def test_sampled_compressor_datedata2():
 
     ps_pd = np.asarray([[68.04, 111.90], [49.5160, 59.4977]])
 
-    convex_hull = ConvexHull(df[variable_headers][:].values)
+    convex_hull = ConvexHull(df[variable_headers].to_numpy())
     lower_rate_qh = get_lower_upper_qhull(convex_hull, axis=0)
 
     lower_convex_hull_0 = lower_rate_qh[0]