Merge branch 'main' into selectn_series_perf_impact

Author: Jeffrharr

.github/workflows/wheels.yml

@@ -153,7 +153,7 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/[email protected].0
+        uses: pypa/[email protected].1
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:

asv_bench/benchmarks/rolling.py

@@ -10,7 +10,19 @@ class Methods:
         ["DataFrame", "Series"],
         [("rolling", {"window": 10}), ("rolling", {"window": 1000}), ("expanding", {})],
         ["int", "float"],
-        ["median", "mean", "max", "min", "std", "count", "skew", "kurt", "sum", "sem"],
+        [
+            "median",
+            "mean",
+            "max",
+            "min",
+            "std",
+            "count",
+            "skew",
+            "kurt",
+            "sum",
+            "sem",
+            "nunique",
+        ],
     )
     param_names = ["constructor", "window_kwargs", "dtype", "method"]
 

ci/code_checks.sh

@@ -72,17 +72,23 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
         -i "pandas.Period.freq GL08" \
         -i "pandas.Period.ordinal GL08" \
-        -i "pandas.Timedelta.max PR02" \
-        -i "pandas.Timedelta.min PR02" \
-        -i "pandas.Timedelta.resolution PR02" \
         -i "pandas.Timestamp.max PR02" \
         -i "pandas.Timestamp.min PR02" \
         -i "pandas.Timestamp.resolution PR02" \
-        -i "pandas.Timestamp.tzinfo GL08" \
         -i "pandas.core.groupby.DataFrameGroupBy.plot PR02" \
         -i "pandas.core.groupby.SeriesGroupBy.plot PR02" \
         -i "pandas.core.resample.Resampler.quantile PR01,PR07" \
         -i "pandas.tseries.offsets.BDay PR02,SA01" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.n GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.normalize GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.rule_code GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.startingMonth GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.n GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.normalize GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.rule_code GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.startingMonth GL08" \
         -i "pandas.tseries.offsets.BQuarterBegin.is_on_offset GL08" \
         -i "pandas.tseries.offsets.BQuarterBegin.n GL08" \
         -i "pandas.tseries.offsets.BQuarterBegin.normalize GL08" \
@@ -185,6 +191,16 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.tseries.offsets.FY5253Quarter.variation GL08" \
         -i "pandas.tseries.offsets.FY5253Quarter.weekday GL08" \
         -i "pandas.tseries.offsets.FY5253Quarter.year_has_extra_week GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.n GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.normalize GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.rule_code GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.startingMonth GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.n GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.normalize GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.rule_code GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.startingMonth GL08" \
         -i "pandas.tseries.offsets.Hour.is_on_offset GL08" \
         -i "pandas.tseries.offsets.Hour.n GL08" \
         -i "pandas.tseries.offsets.Hour.normalize GL08" \

doc/source/reference/offset_frequency.rst

@@ -776,6 +776,146 @@ Methods
     QuarterBegin.is_year_start
     QuarterBegin.is_year_end
 
+BHalfYearEnd
+------------
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearEnd
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearEnd.freqstr
+    BHalfYearEnd.kwds
+    BHalfYearEnd.name
+    BHalfYearEnd.nanos
+    BHalfYearEnd.normalize
+    BHalfYearEnd.rule_code
+    BHalfYearEnd.n
+    BHalfYearEnd.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearEnd.copy
+    BHalfYearEnd.is_on_offset
+    BHalfYearEnd.is_month_start
+    BHalfYearEnd.is_month_end
+    BHalfYearEnd.is_quarter_start
+    BHalfYearEnd.is_quarter_end
+    BHalfYearEnd.is_year_start
+    BHalfYearEnd.is_year_end
+
+BHalfYearBegin
+--------------
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearBegin
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearBegin.freqstr
+    BHalfYearBegin.kwds
+    BHalfYearBegin.name
+    BHalfYearBegin.nanos
+    BHalfYearBegin.normalize
+    BHalfYearBegin.rule_code
+    BHalfYearBegin.n
+    BHalfYearBegin.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearBegin.copy
+    BHalfYearBegin.is_on_offset
+    BHalfYearBegin.is_month_start
+    BHalfYearBegin.is_month_end
+    BHalfYearBegin.is_quarter_start
+    BHalfYearBegin.is_quarter_end
+    BHalfYearBegin.is_year_start
+    BHalfYearBegin.is_year_end
+
+HalfYearEnd
+-----------
+.. autosummary::
+   :toctree: api/
+
+    HalfYearEnd
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearEnd.freqstr
+    HalfYearEnd.kwds
+    HalfYearEnd.name
+    HalfYearEnd.nanos
+    HalfYearEnd.normalize
+    HalfYearEnd.rule_code
+    HalfYearEnd.n
+    HalfYearEnd.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearEnd.copy
+    HalfYearEnd.is_on_offset
+    HalfYearEnd.is_month_start
+    HalfYearEnd.is_month_end
+    HalfYearEnd.is_quarter_start
+    HalfYearEnd.is_quarter_end
+    HalfYearEnd.is_year_start
+    HalfYearEnd.is_year_end
+
+HalfYearBegin
+-------------
+.. autosummary::
+   :toctree: api/
+
+    HalfYearBegin
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearBegin.freqstr
+    HalfYearBegin.kwds
+    HalfYearBegin.name
+    HalfYearBegin.nanos
+    HalfYearBegin.normalize
+    HalfYearBegin.rule_code
+    HalfYearBegin.n
+    HalfYearBegin.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearBegin.copy
+    HalfYearBegin.is_on_offset
+    HalfYearBegin.is_month_start
+    HalfYearBegin.is_month_end
+    HalfYearBegin.is_quarter_start
+    HalfYearBegin.is_quarter_end
+    HalfYearBegin.is_year_start
+    HalfYearBegin.is_year_end
+
 BYearEnd
 --------
 .. autosummary::

doc/source/reference/window.rst

@@ -42,6 +42,7 @@ Rolling window functions
    Rolling.quantile
    Rolling.sem
    Rolling.rank
+   Rolling.nunique
 
 .. _api.functions_window:
 
@@ -86,6 +87,7 @@ Expanding window functions
    Expanding.quantile
    Expanding.sem
    Expanding.rank
+   Expanding.nunique
 
 .. _api.functions_ewm:
 

doc/source/user_guide/timeseries.rst

@@ -891,6 +891,10 @@ into ``freq`` keyword arguments. The available date offsets and associated frequ
     :class:`~pandas.tseries.offsets.BQuarterEnd`, ``'BQE``, "business quarter end"
     :class:`~pandas.tseries.offsets.BQuarterBegin`, ``'BQS'``, "business quarter begin"
     :class:`~pandas.tseries.offsets.FY5253Quarter`, ``'REQ'``, "retail (aka 52-53 week) quarter"
+    :class:`~pandas.tseries.offsets.HalfYearEnd`, ``'HYE'``, "calendar half year end"
+    :class:`~pandas.tseries.offsets.HalfYearBegin`, ``'HYS'``, "calendar half year begin"
+    :class:`~pandas.tseries.offsets.BHalfYearEnd`, ``'BHYE``, "business half year end"
+    :class:`~pandas.tseries.offsets.BHalfYearBegin`, ``'BHYS'``, "business half year begin"
     :class:`~pandas.tseries.offsets.YearEnd`, ``'YE'``, "calendar year end"
     :class:`~pandas.tseries.offsets.YearBegin`, ``'YS'`` or ``'BYS'``,"calendar year begin"
     :class:`~pandas.tseries.offsets.BYearEnd`, ``'BYE'``, "business year end"

doc/source/whatsnew/v2.3.0.rst

@@ -37,6 +37,7 @@ Other enhancements
   updated to work correctly with NumPy >= 2 (:issue:`57739`)
 - :meth:`Series.str.decode` result now has ``StringDtype`` when ``future.infer_string`` is True (:issue:`60709`)
 - :meth:`~Series.to_hdf` and :meth:`~DataFrame.to_hdf` now round-trip with ``StringDtype``  (:issue:`60663`)
+- Improved ``repr`` of :class:`.NumpyExtensionArray` to account for NEP51 (:issue:`61085`)
 - The :meth:`Series.str.decode` has gained the argument ``dtype`` to control the dtype of the result (:issue:`60940`)
 - The :meth:`~Series.cumsum`, :meth:`~Series.cummin`, and :meth:`~Series.cummax` reductions are now implemented for ``StringDtype`` columns (:issue:`60633`)
 - The :meth:`~Series.sum` reduction is now implemented for ``StringDtype`` columns (:issue:`59853`)
@@ -118,6 +119,7 @@ Conversion
 
 Strings
 ^^^^^^^
+- Bug in :meth:`DataFrame.sum` with ``axis=1``, :meth:`.DataFrameGroupBy.sum` or :meth:`.SeriesGroupBy.sum` with ``skipna=True``, and :meth:`.Resampler.sum` on :class:`StringDtype` with all NA values resulted in ``0`` and is now the empty string ``""`` (:issue:`60229`)
 - Bug in :meth:`Series.__pos__` and :meth:`DataFrame.__pos__` did not raise for :class:`StringDtype` with ``storage="pyarrow"`` (:issue:`60710`)
 - Bug in :meth:`Series.rank` for :class:`StringDtype` with ``storage="pyarrow"`` incorrectly returning integer results in case of ``method="average"`` and raising an error if it would truncate results (:issue:`59768`)
 - Bug in :meth:`Series.replace` with :class:`StringDtype` when replacing with a non-string value was not upcasting to ``object`` dtype (:issue:`60282`)

doc/source/whatsnew/v3.0.0.rst

@@ -61,10 +61,13 @@ Other enhancements
 - :meth:`Series.cummin` and :meth:`Series.cummax` now supports :class:`CategoricalDtype` (:issue:`52335`)
 - :meth:`Series.plot` now correctly handle the ``ylabel`` parameter for pie charts, allowing for explicit control over the y-axis label (:issue:`58239`)
 - :meth:`DataFrame.plot.scatter` argument ``c`` now accepts a column of strings, where rows with the same string are colored identically (:issue:`16827` and :issue:`16485`)
+- :class:`ArrowDtype` now supports ``pyarrow.JsonType`` (:issue:`60958`)
 - :class:`DataFrameGroupBy` and :class:`SeriesGroupBy` methods ``sum``, ``mean``, ``median``, ``prod``, ``min``, ``max``, ``std``, ``var`` and ``sem`` now accept ``skipna`` parameter (:issue:`15675`)
+- :class:`Rolling` and :class:`Expanding` now support ``nunique`` (:issue:`26958`)
 - :class:`Rolling` and :class:`Expanding` now support aggregations ``first`` and ``last`` (:issue:`33155`)
 - :func:`read_parquet` accepts ``to_pandas_kwargs`` which are forwarded to :meth:`pyarrow.Table.to_pandas` which enables passing additional keywords to customize the conversion to pandas, such as ``maps_as_pydicts`` to read the Parquet map data type as python dictionaries (:issue:`56842`)
 - :meth:`.DataFrameGroupBy.transform`, :meth:`.SeriesGroupBy.transform`, :meth:`.DataFrameGroupBy.agg`, :meth:`.SeriesGroupBy.agg`, :meth:`.SeriesGroupBy.apply`, :meth:`.DataFrameGroupBy.apply` now support ``kurt`` (:issue:`40139`)
+- :meth:`DataFrame.apply` supports using third-party execution engines like the Bodo.ai JIT compiler (:issue:`60668`)
 - :meth:`DataFrameGroupBy.transform`, :meth:`SeriesGroupBy.transform`, :meth:`DataFrameGroupBy.agg`, :meth:`SeriesGroupBy.agg`, :meth:`RollingGroupby.apply`, :meth:`ExpandingGroupby.apply`, :meth:`Rolling.apply`, :meth:`Expanding.apply`, :meth:`DataFrame.apply` with ``engine="numba"`` now supports positional arguments passed as kwargs (:issue:`58995`)
 - :meth:`Rolling.agg`, :meth:`Expanding.agg` and :meth:`ExponentialMovingWindow.agg` now accept :class:`NamedAgg` aggregations through ``**kwargs`` (:issue:`28333`)
 - :meth:`Series.map` can now accept kwargs to pass on to func (:issue:`59814`)
@@ -636,6 +639,7 @@ Bug fixes
 Categorical
 ^^^^^^^^^^^
 - Bug in :func:`Series.apply` where ``nan`` was ignored for :class:`CategoricalDtype` (:issue:`59938`)
+- Bug in :meth:`Series.convert_dtypes` with ``dtype_backend="pyarrow"`` where empty :class:`CategoricalDtype` :class:`Series` raised an error or got converted to ``null[pyarrow]`` (:issue:`59934`)
 -
 
 Datetimelike
@@ -697,11 +701,13 @@ Interval
 Indexing
 ^^^^^^^^
 - Bug in :meth:`DataFrame.__getitem__` returning modified columns when called with ``slice`` in Python 3.12 (:issue:`57500`)
+- Bug in :meth:`DataFrame.__getitem__` when slicing a :class:`DataFrame` with many rows raised an ``OverflowError`` (:issue:`59531`)
 - Bug in :meth:`DataFrame.from_records` throwing a ``ValueError`` when passed an empty list in ``index`` (:issue:`58594`)
 - Bug in :meth:`DataFrame.loc` with inconsistent behavior of loc-set with 2 given indexes to Series (:issue:`59933`)
 - Bug in :meth:`Index.get_indexer` and similar methods when ``NaN`` is located at or after position 128 (:issue:`58924`)
 - Bug in :meth:`MultiIndex.insert` when a new value inserted to a datetime-like level gets cast to ``NaT`` and fails indexing (:issue:`60388`)
 - Bug in printing :attr:`Index.names` and :attr:`MultiIndex.levels` would not escape single quotes (:issue:`60190`)
+- Bug in reindexing of :class:`DataFrame` with :class:`PeriodDtype` columns in case of consolidated block (:issue:`60980`, :issue:`60273`)
 
 Missing
 ^^^^^^^
@@ -716,7 +722,7 @@ MultiIndex
 - :func:`MultiIndex.get_level_values` accessing a :class:`DatetimeIndex` does not carry the frequency attribute along (:issue:`58327`, :issue:`57949`)
 - Bug in :class:`DataFrame` arithmetic operations in case of unaligned MultiIndex columns (:issue:`60498`)
 - Bug in :class:`DataFrame` arithmetic operations with :class:`Series` in case of unaligned MultiIndex (:issue:`61009`)
--
+- Bug in :meth:`MultiIndex.from_tuples` causing wrong output with input of type tuples having NaN values (:issue:`60695`, :issue:`60988`)
 
 I/O
 ^^^
@@ -738,6 +744,7 @@ I/O
 - Bug in :meth:`read_csv` where the order of the ``na_values`` makes an inconsistency when ``na_values`` is a list non-string values. (:issue:`59303`)
 - Bug in :meth:`read_excel` raising ``ValueError`` when passing array of boolean values when ``dtype="boolean"``. (:issue:`58159`)
 - Bug in :meth:`read_html` where ``rowspan`` in header row causes incorrect conversion to ``DataFrame``. (:issue:`60210`)
+- Bug in :meth:`read_json` ignoring the given ``dtype`` when ``engine="pyarrow"`` (:issue:`59516`)
 - Bug in :meth:`read_json` not validating the ``typ`` argument to not be exactly ``"frame"`` or ``"series"`` (:issue:`59124`)
 - Bug in :meth:`read_json` where extreme value integers in string format were incorrectly parsed as a different integer number (:issue:`20608`)
 - Bug in :meth:`read_stata` raising ``KeyError`` when input file is stored in big-endian format and contains strL data. (:issue:`58638`)
@@ -783,6 +790,7 @@ Reshaping
 ^^^^^^^^^
 - Bug in :func:`qcut` where values at the quantile boundaries could be incorrectly assigned (:issue:`59355`)
 - Bug in :meth:`DataFrame.combine_first` not preserving the column order (:issue:`60427`)
+- Bug in :meth:`DataFrame.explode` producing incorrect result for :class:`pyarrow.large_list` type (:issue:`61091`)
 - Bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`)
 - Bug in :meth:`DataFrame.join` when a :class:`DataFrame` with a :class:`MultiIndex` would raise an ``AssertionError`` when :attr:`MultiIndex.names` contained ``None``. (:issue:`58721`)
 - Bug in :meth:`DataFrame.merge` where merging on a column containing only ``NaN`` values resulted in an out-of-bounds array access (:issue:`59421`)
@@ -836,6 +844,7 @@ Other
 - Bug in :meth:`Series.diff` allowing non-integer values for the ``periods`` argument. (:issue:`56607`)
 - Bug in :meth:`Series.dt` methods in :class:`ArrowDtype` that were returning incorrect values. (:issue:`57355`)
 - Bug in :meth:`Series.isin` raising ``TypeError`` when series is large (>10**6) and ``values`` contains NA (:issue:`60678`)
+- Bug in :meth:`Series.mode` where an exception was raised when taking the mode with nullable types with no null values in the series. (:issue:`58926`)
 - Bug in :meth:`Series.rank` that doesn't preserve missing values for nullable integers when ``na_option='keep'``. (:issue:`56976`)
 - Bug in :meth:`Series.replace` and :meth:`DataFrame.replace` inconsistently replacing matching instances when ``regex=True`` and missing values are present. (:issue:`56599`)
 - Bug in :meth:`Series.replace` and :meth:`DataFrame.replace` throwing ``ValueError`` when ``regex=True`` and all NA values. (:issue:`60688`)

pandas/_libs/hashtable_func_helper.pxi.in

@@ -430,7 +430,7 @@ def mode(ndarray[htfunc_t] values, bint dropna, const uint8_t[:] mask=None):
 
     if na_counter > 0:
         res_mask = np.zeros(j+1, dtype=np.bool_)
-        res_mask[j] = True
+        res_mask[j] = (na_counter == max_count)
     return modes[:j + 1], res_mask
 
 

pandas/_libs/lib.pyx

@@ -502,7 +502,7 @@ def has_only_ints_or_nan(const floating[:] arr) -> bool:
     return True
 
 
-def maybe_indices_to_slice(ndarray[intp_t, ndim=1] indices, int max_len):
+def maybe_indices_to_slice(ndarray[intp_t, ndim=1] indices, intp_t max_len):
     cdef:
         Py_ssize_t i, n = len(indices)
         intp_t k, vstart, vlast, v
@@ -1518,7 +1518,7 @@ cdef object _try_infer_map(object dtype):
 
 def infer_dtype(value: object, skipna: bool = True) -> str:
     """
-    Return a string label of the type of a scalar or list-like of values.
+    Return a string label of the type of the elements in a list-like input.
 
     This method inspects the elements of the provided input and determines
     classification of its data type. It is particularly useful for
@@ -1527,7 +1527,7 @@ def infer_dtype(value: object, skipna: bool = True) -> str:
 
     Parameters
     ----------
-    value : scalar, list, ndarray, or pandas type
+    value : list, ndarray, or pandas type
         The input data to infer the dtype.
     skipna : bool, default True
         Ignore NaN values when inferring the type.