Skip to content

Commit d10aa50

Browse files
authored
Merge branch 'main' into selectn_series_perf_impact
2 parents 5e82306 + 64e8f2c commit d10aa50

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+1340
-170
lines changed

.github/workflows/wheels.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ jobs:
153153
run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
154154

155155
- name: Build wheels
156-
uses: pypa/[email protected].0
156+
uses: pypa/[email protected].1
157157
with:
158158
package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
159159
env:

asv_bench/benchmarks/rolling.py

+13-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,19 @@ class Methods:
1010
["DataFrame", "Series"],
1111
[("rolling", {"window": 10}), ("rolling", {"window": 1000}), ("expanding", {})],
1212
["int", "float"],
13-
["median", "mean", "max", "min", "std", "count", "skew", "kurt", "sum", "sem"],
13+
[
14+
"median",
15+
"mean",
16+
"max",
17+
"min",
18+
"std",
19+
"count",
20+
"skew",
21+
"kurt",
22+
"sum",
23+
"sem",
24+
"nunique",
25+
],
1426
)
1527
param_names = ["constructor", "window_kwargs", "dtype", "method"]
1628

ci/code_checks.sh

+20-4
Original file line numberDiff line numberDiff line change
@@ -72,17 +72,23 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
7272
-i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
7373
-i "pandas.Period.freq GL08" \
7474
-i "pandas.Period.ordinal GL08" \
75-
-i "pandas.Timedelta.max PR02" \
76-
-i "pandas.Timedelta.min PR02" \
77-
-i "pandas.Timedelta.resolution PR02" \
7875
-i "pandas.Timestamp.max PR02" \
7976
-i "pandas.Timestamp.min PR02" \
8077
-i "pandas.Timestamp.resolution PR02" \
81-
-i "pandas.Timestamp.tzinfo GL08" \
8278
-i "pandas.core.groupby.DataFrameGroupBy.plot PR02" \
8379
-i "pandas.core.groupby.SeriesGroupBy.plot PR02" \
8480
-i "pandas.core.resample.Resampler.quantile PR01,PR07" \
8581
-i "pandas.tseries.offsets.BDay PR02,SA01" \
82+
-i "pandas.tseries.offsets.BHalfYearBegin.is_on_offset GL08" \
83+
-i "pandas.tseries.offsets.BHalfYearBegin.n GL08" \
84+
-i "pandas.tseries.offsets.BHalfYearBegin.normalize GL08" \
85+
-i "pandas.tseries.offsets.BHalfYearBegin.rule_code GL08" \
86+
-i "pandas.tseries.offsets.BHalfYearBegin.startingMonth GL08" \
87+
-i "pandas.tseries.offsets.BHalfYearEnd.is_on_offset GL08" \
88+
-i "pandas.tseries.offsets.BHalfYearEnd.n GL08" \
89+
-i "pandas.tseries.offsets.BHalfYearEnd.normalize GL08" \
90+
-i "pandas.tseries.offsets.BHalfYearEnd.rule_code GL08" \
91+
-i "pandas.tseries.offsets.BHalfYearEnd.startingMonth GL08" \
8692
-i "pandas.tseries.offsets.BQuarterBegin.is_on_offset GL08" \
8793
-i "pandas.tseries.offsets.BQuarterBegin.n GL08" \
8894
-i "pandas.tseries.offsets.BQuarterBegin.normalize GL08" \
@@ -185,6 +191,16 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
185191
-i "pandas.tseries.offsets.FY5253Quarter.variation GL08" \
186192
-i "pandas.tseries.offsets.FY5253Quarter.weekday GL08" \
187193
-i "pandas.tseries.offsets.FY5253Quarter.year_has_extra_week GL08" \
194+
-i "pandas.tseries.offsets.HalfYearBegin.is_on_offset GL08" \
195+
-i "pandas.tseries.offsets.HalfYearBegin.n GL08" \
196+
-i "pandas.tseries.offsets.HalfYearBegin.normalize GL08" \
197+
-i "pandas.tseries.offsets.HalfYearBegin.rule_code GL08" \
198+
-i "pandas.tseries.offsets.HalfYearBegin.startingMonth GL08" \
199+
-i "pandas.tseries.offsets.HalfYearEnd.is_on_offset GL08" \
200+
-i "pandas.tseries.offsets.HalfYearEnd.n GL08" \
201+
-i "pandas.tseries.offsets.HalfYearEnd.normalize GL08" \
202+
-i "pandas.tseries.offsets.HalfYearEnd.rule_code GL08" \
203+
-i "pandas.tseries.offsets.HalfYearEnd.startingMonth GL08" \
188204
-i "pandas.tseries.offsets.Hour.is_on_offset GL08" \
189205
-i "pandas.tseries.offsets.Hour.n GL08" \
190206
-i "pandas.tseries.offsets.Hour.normalize GL08" \

doc/source/reference/offset_frequency.rst

+140
Original file line numberDiff line numberDiff line change
@@ -776,6 +776,146 @@ Methods
776776
QuarterBegin.is_year_start
777777
QuarterBegin.is_year_end
778778

779+
BHalfYearEnd
780+
------------
781+
.. autosummary::
782+
:toctree: api/
783+
784+
BHalfYearEnd
785+
786+
Properties
787+
~~~~~~~~~~
788+
.. autosummary::
789+
:toctree: api/
790+
791+
BHalfYearEnd.freqstr
792+
BHalfYearEnd.kwds
793+
BHalfYearEnd.name
794+
BHalfYearEnd.nanos
795+
BHalfYearEnd.normalize
796+
BHalfYearEnd.rule_code
797+
BHalfYearEnd.n
798+
BHalfYearEnd.startingMonth
799+
800+
Methods
801+
~~~~~~~
802+
.. autosummary::
803+
:toctree: api/
804+
805+
BHalfYearEnd.copy
806+
BHalfYearEnd.is_on_offset
807+
BHalfYearEnd.is_month_start
808+
BHalfYearEnd.is_month_end
809+
BHalfYearEnd.is_quarter_start
810+
BHalfYearEnd.is_quarter_end
811+
BHalfYearEnd.is_year_start
812+
BHalfYearEnd.is_year_end
813+
814+
BHalfYearBegin
815+
--------------
816+
.. autosummary::
817+
:toctree: api/
818+
819+
BHalfYearBegin
820+
821+
Properties
822+
~~~~~~~~~~
823+
.. autosummary::
824+
:toctree: api/
825+
826+
BHalfYearBegin.freqstr
827+
BHalfYearBegin.kwds
828+
BHalfYearBegin.name
829+
BHalfYearBegin.nanos
830+
BHalfYearBegin.normalize
831+
BHalfYearBegin.rule_code
832+
BHalfYearBegin.n
833+
BHalfYearBegin.startingMonth
834+
835+
Methods
836+
~~~~~~~
837+
.. autosummary::
838+
:toctree: api/
839+
840+
BHalfYearBegin.copy
841+
BHalfYearBegin.is_on_offset
842+
BHalfYearBegin.is_month_start
843+
BHalfYearBegin.is_month_end
844+
BHalfYearBegin.is_quarter_start
845+
BHalfYearBegin.is_quarter_end
846+
BHalfYearBegin.is_year_start
847+
BHalfYearBegin.is_year_end
848+
849+
HalfYearEnd
850+
-----------
851+
.. autosummary::
852+
:toctree: api/
853+
854+
HalfYearEnd
855+
856+
Properties
857+
~~~~~~~~~~
858+
.. autosummary::
859+
:toctree: api/
860+
861+
HalfYearEnd.freqstr
862+
HalfYearEnd.kwds
863+
HalfYearEnd.name
864+
HalfYearEnd.nanos
865+
HalfYearEnd.normalize
866+
HalfYearEnd.rule_code
867+
HalfYearEnd.n
868+
HalfYearEnd.startingMonth
869+
870+
Methods
871+
~~~~~~~
872+
.. autosummary::
873+
:toctree: api/
874+
875+
HalfYearEnd.copy
876+
HalfYearEnd.is_on_offset
877+
HalfYearEnd.is_month_start
878+
HalfYearEnd.is_month_end
879+
HalfYearEnd.is_quarter_start
880+
HalfYearEnd.is_quarter_end
881+
HalfYearEnd.is_year_start
882+
HalfYearEnd.is_year_end
883+
884+
HalfYearBegin
885+
-------------
886+
.. autosummary::
887+
:toctree: api/
888+
889+
HalfYearBegin
890+
891+
Properties
892+
~~~~~~~~~~
893+
.. autosummary::
894+
:toctree: api/
895+
896+
HalfYearBegin.freqstr
897+
HalfYearBegin.kwds
898+
HalfYearBegin.name
899+
HalfYearBegin.nanos
900+
HalfYearBegin.normalize
901+
HalfYearBegin.rule_code
902+
HalfYearBegin.n
903+
HalfYearBegin.startingMonth
904+
905+
Methods
906+
~~~~~~~
907+
.. autosummary::
908+
:toctree: api/
909+
910+
HalfYearBegin.copy
911+
HalfYearBegin.is_on_offset
912+
HalfYearBegin.is_month_start
913+
HalfYearBegin.is_month_end
914+
HalfYearBegin.is_quarter_start
915+
HalfYearBegin.is_quarter_end
916+
HalfYearBegin.is_year_start
917+
HalfYearBegin.is_year_end
918+
779919
BYearEnd
780920
--------
781921
.. autosummary::

doc/source/reference/window.rst

+2
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ Rolling window functions
4242
Rolling.quantile
4343
Rolling.sem
4444
Rolling.rank
45+
Rolling.nunique
4546

4647
.. _api.functions_window:
4748

@@ -86,6 +87,7 @@ Expanding window functions
8687
Expanding.quantile
8788
Expanding.sem
8889
Expanding.rank
90+
Expanding.nunique
8991

9092
.. _api.functions_ewm:
9193

doc/source/user_guide/timeseries.rst

+4
Original file line numberDiff line numberDiff line change
@@ -891,6 +891,10 @@ into ``freq`` keyword arguments. The available date offsets and associated frequ
891891
:class:`~pandas.tseries.offsets.BQuarterEnd`, ``'BQE``, "business quarter end"
892892
:class:`~pandas.tseries.offsets.BQuarterBegin`, ``'BQS'``, "business quarter begin"
893893
:class:`~pandas.tseries.offsets.FY5253Quarter`, ``'REQ'``, "retail (aka 52-53 week) quarter"
894+
:class:`~pandas.tseries.offsets.HalfYearEnd`, ``'HYE'``, "calendar half year end"
895+
:class:`~pandas.tseries.offsets.HalfYearBegin`, ``'HYS'``, "calendar half year begin"
896+
:class:`~pandas.tseries.offsets.BHalfYearEnd`, ``'BHYE``, "business half year end"
897+
:class:`~pandas.tseries.offsets.BHalfYearBegin`, ``'BHYS'``, "business half year begin"
894898
:class:`~pandas.tseries.offsets.YearEnd`, ``'YE'``, "calendar year end"
895899
:class:`~pandas.tseries.offsets.YearBegin`, ``'YS'`` or ``'BYS'``,"calendar year begin"
896900
:class:`~pandas.tseries.offsets.BYearEnd`, ``'BYE'``, "business year end"

doc/source/whatsnew/v2.3.0.rst

+2
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ Other enhancements
3737
updated to work correctly with NumPy >= 2 (:issue:`57739`)
3838
- :meth:`Series.str.decode` result now has ``StringDtype`` when ``future.infer_string`` is True (:issue:`60709`)
3939
- :meth:`~Series.to_hdf` and :meth:`~DataFrame.to_hdf` now round-trip with ``StringDtype`` (:issue:`60663`)
40+
- Improved ``repr`` of :class:`.NumpyExtensionArray` to account for NEP51 (:issue:`61085`)
4041
- The :meth:`Series.str.decode` has gained the argument ``dtype`` to control the dtype of the result (:issue:`60940`)
4142
- The :meth:`~Series.cumsum`, :meth:`~Series.cummin`, and :meth:`~Series.cummax` reductions are now implemented for ``StringDtype`` columns (:issue:`60633`)
4243
- The :meth:`~Series.sum` reduction is now implemented for ``StringDtype`` columns (:issue:`59853`)
@@ -118,6 +119,7 @@ Conversion
118119

119120
Strings
120121
^^^^^^^
122+
- Bug in :meth:`DataFrame.sum` with ``axis=1``, :meth:`.DataFrameGroupBy.sum` or :meth:`.SeriesGroupBy.sum` with ``skipna=True``, and :meth:`.Resampler.sum` on :class:`StringDtype` with all NA values resulted in ``0`` and is now the empty string ``""`` (:issue:`60229`)
121123
- Bug in :meth:`Series.__pos__` and :meth:`DataFrame.__pos__` did not raise for :class:`StringDtype` with ``storage="pyarrow"`` (:issue:`60710`)
122124
- Bug in :meth:`Series.rank` for :class:`StringDtype` with ``storage="pyarrow"`` incorrectly returning integer results in case of ``method="average"`` and raising an error if it would truncate results (:issue:`59768`)
123125
- Bug in :meth:`Series.replace` with :class:`StringDtype` when replacing with a non-string value was not upcasting to ``object`` dtype (:issue:`60282`)

doc/source/whatsnew/v3.0.0.rst

+10-1
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,13 @@ Other enhancements
6161
- :meth:`Series.cummin` and :meth:`Series.cummax` now supports :class:`CategoricalDtype` (:issue:`52335`)
6262
- :meth:`Series.plot` now correctly handle the ``ylabel`` parameter for pie charts, allowing for explicit control over the y-axis label (:issue:`58239`)
6363
- :meth:`DataFrame.plot.scatter` argument ``c`` now accepts a column of strings, where rows with the same string are colored identically (:issue:`16827` and :issue:`16485`)
64+
- :class:`ArrowDtype` now supports ``pyarrow.JsonType`` (:issue:`60958`)
6465
- :class:`DataFrameGroupBy` and :class:`SeriesGroupBy` methods ``sum``, ``mean``, ``median``, ``prod``, ``min``, ``max``, ``std``, ``var`` and ``sem`` now accept ``skipna`` parameter (:issue:`15675`)
66+
- :class:`Rolling` and :class:`Expanding` now support ``nunique`` (:issue:`26958`)
6567
- :class:`Rolling` and :class:`Expanding` now support aggregations ``first`` and ``last`` (:issue:`33155`)
6668
- :func:`read_parquet` accepts ``to_pandas_kwargs`` which are forwarded to :meth:`pyarrow.Table.to_pandas` which enables passing additional keywords to customize the conversion to pandas, such as ``maps_as_pydicts`` to read the Parquet map data type as python dictionaries (:issue:`56842`)
6769
- :meth:`.DataFrameGroupBy.transform`, :meth:`.SeriesGroupBy.transform`, :meth:`.DataFrameGroupBy.agg`, :meth:`.SeriesGroupBy.agg`, :meth:`.SeriesGroupBy.apply`, :meth:`.DataFrameGroupBy.apply` now support ``kurt`` (:issue:`40139`)
70+
- :meth:`DataFrame.apply` supports using third-party execution engines like the Bodo.ai JIT compiler (:issue:`60668`)
6871
- :meth:`DataFrameGroupBy.transform`, :meth:`SeriesGroupBy.transform`, :meth:`DataFrameGroupBy.agg`, :meth:`SeriesGroupBy.agg`, :meth:`RollingGroupby.apply`, :meth:`ExpandingGroupby.apply`, :meth:`Rolling.apply`, :meth:`Expanding.apply`, :meth:`DataFrame.apply` with ``engine="numba"`` now supports positional arguments passed as kwargs (:issue:`58995`)
6972
- :meth:`Rolling.agg`, :meth:`Expanding.agg` and :meth:`ExponentialMovingWindow.agg` now accept :class:`NamedAgg` aggregations through ``**kwargs`` (:issue:`28333`)
7073
- :meth:`Series.map` can now accept kwargs to pass on to func (:issue:`59814`)
@@ -636,6 +639,7 @@ Bug fixes
636639
Categorical
637640
^^^^^^^^^^^
638641
- Bug in :func:`Series.apply` where ``nan`` was ignored for :class:`CategoricalDtype` (:issue:`59938`)
642+
- Bug in :meth:`Series.convert_dtypes` with ``dtype_backend="pyarrow"`` where empty :class:`CategoricalDtype` :class:`Series` raised an error or got converted to ``null[pyarrow]`` (:issue:`59934`)
639643
-
640644

641645
Datetimelike
@@ -697,11 +701,13 @@ Interval
697701
Indexing
698702
^^^^^^^^
699703
- Bug in :meth:`DataFrame.__getitem__` returning modified columns when called with ``slice`` in Python 3.12 (:issue:`57500`)
704+
- Bug in :meth:`DataFrame.__getitem__` when slicing a :class:`DataFrame` with many rows raised an ``OverflowError`` (:issue:`59531`)
700705
- Bug in :meth:`DataFrame.from_records` throwing a ``ValueError`` when passed an empty list in ``index`` (:issue:`58594`)
701706
- Bug in :meth:`DataFrame.loc` with inconsistent behavior of loc-set with 2 given indexes to Series (:issue:`59933`)
702707
- Bug in :meth:`Index.get_indexer` and similar methods when ``NaN`` is located at or after position 128 (:issue:`58924`)
703708
- Bug in :meth:`MultiIndex.insert` when a new value inserted to a datetime-like level gets cast to ``NaT`` and fails indexing (:issue:`60388`)
704709
- Bug in printing :attr:`Index.names` and :attr:`MultiIndex.levels` would not escape single quotes (:issue:`60190`)
710+
- Bug in reindexing of :class:`DataFrame` with :class:`PeriodDtype` columns in case of consolidated block (:issue:`60980`, :issue:`60273`)
705711

706712
Missing
707713
^^^^^^^
@@ -716,7 +722,7 @@ MultiIndex
716722
- :func:`MultiIndex.get_level_values` accessing a :class:`DatetimeIndex` does not carry the frequency attribute along (:issue:`58327`, :issue:`57949`)
717723
- Bug in :class:`DataFrame` arithmetic operations in case of unaligned MultiIndex columns (:issue:`60498`)
718724
- Bug in :class:`DataFrame` arithmetic operations with :class:`Series` in case of unaligned MultiIndex (:issue:`61009`)
719-
-
725+
- Bug in :meth:`MultiIndex.from_tuples` causing wrong output with input of type tuples having NaN values (:issue:`60695`, :issue:`60988`)
720726

721727
I/O
722728
^^^
@@ -738,6 +744,7 @@ I/O
738744
- Bug in :meth:`read_csv` where the order of the ``na_values`` makes an inconsistency when ``na_values`` is a list non-string values. (:issue:`59303`)
739745
- Bug in :meth:`read_excel` raising ``ValueError`` when passing array of boolean values when ``dtype="boolean"``. (:issue:`58159`)
740746
- Bug in :meth:`read_html` where ``rowspan`` in header row causes incorrect conversion to ``DataFrame``. (:issue:`60210`)
747+
- Bug in :meth:`read_json` ignoring the given ``dtype`` when ``engine="pyarrow"`` (:issue:`59516`)
741748
- Bug in :meth:`read_json` not validating the ``typ`` argument to not be exactly ``"frame"`` or ``"series"`` (:issue:`59124`)
742749
- Bug in :meth:`read_json` where extreme value integers in string format were incorrectly parsed as a different integer number (:issue:`20608`)
743750
- Bug in :meth:`read_stata` raising ``KeyError`` when input file is stored in big-endian format and contains strL data. (:issue:`58638`)
@@ -783,6 +790,7 @@ Reshaping
783790
^^^^^^^^^
784791
- Bug in :func:`qcut` where values at the quantile boundaries could be incorrectly assigned (:issue:`59355`)
785792
- Bug in :meth:`DataFrame.combine_first` not preserving the column order (:issue:`60427`)
793+
- Bug in :meth:`DataFrame.explode` producing incorrect result for :class:`pyarrow.large_list` type (:issue:`61091`)
786794
- Bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`)
787795
- Bug in :meth:`DataFrame.join` when a :class:`DataFrame` with a :class:`MultiIndex` would raise an ``AssertionError`` when :attr:`MultiIndex.names` contained ``None``. (:issue:`58721`)
788796
- Bug in :meth:`DataFrame.merge` where merging on a column containing only ``NaN`` values resulted in an out-of-bounds array access (:issue:`59421`)
@@ -836,6 +844,7 @@ Other
836844
- Bug in :meth:`Series.diff` allowing non-integer values for the ``periods`` argument. (:issue:`56607`)
837845
- Bug in :meth:`Series.dt` methods in :class:`ArrowDtype` that were returning incorrect values. (:issue:`57355`)
838846
- Bug in :meth:`Series.isin` raising ``TypeError`` when series is large (>10**6) and ``values`` contains NA (:issue:`60678`)
847+
- Bug in :meth:`Series.mode` where an exception was raised when taking the mode with nullable types with no null values in the series. (:issue:`58926`)
839848
- Bug in :meth:`Series.rank` that doesn't preserve missing values for nullable integers when ``na_option='keep'``. (:issue:`56976`)
840849
- Bug in :meth:`Series.replace` and :meth:`DataFrame.replace` inconsistently replacing matching instances when ``regex=True`` and missing values are present. (:issue:`56599`)
841850
- Bug in :meth:`Series.replace` and :meth:`DataFrame.replace` throwing ``ValueError`` when ``regex=True`` and all NA values. (:issue:`60688`)

pandas/_libs/hashtable_func_helper.pxi.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@ def mode(ndarray[htfunc_t] values, bint dropna, const uint8_t[:] mask=None):
430430

431431
if na_counter > 0:
432432
res_mask = np.zeros(j+1, dtype=np.bool_)
433-
res_mask[j] = True
433+
res_mask[j] = (na_counter == max_count)
434434
return modes[:j + 1], res_mask
435435

436436

pandas/_libs/lib.pyx

+3-3
Original file line numberDiff line numberDiff line change
@@ -502,7 +502,7 @@ def has_only_ints_or_nan(const floating[:] arr) -> bool:
502502
return True
503503

504504

505-
def maybe_indices_to_slice(ndarray[intp_t, ndim=1] indices, int max_len):
505+
def maybe_indices_to_slice(ndarray[intp_t, ndim=1] indices, intp_t max_len):
506506
cdef:
507507
Py_ssize_t i, n = len(indices)
508508
intp_t k, vstart, vlast, v
@@ -1518,7 +1518,7 @@ cdef object _try_infer_map(object dtype):
15181518

15191519
def infer_dtype(value: object, skipna: bool = True) -> str:
15201520
"""
1521-
Return a string label of the type of a scalar or list-like of values.
1521+
Return a string label of the type of the elements in a list-like input.
15221522

15231523
This method inspects the elements of the provided input and determines
15241524
classification of its data type. It is particularly useful for
@@ -1527,7 +1527,7 @@ def infer_dtype(value: object, skipna: bool = True) -> str:
15271527

15281528
Parameters
15291529
----------
1530-
value : scalar, list, ndarray, or pandas type
1530+
value : list, ndarray, or pandas type
15311531
The input data to infer the dtype.
15321532
skipna : bool, default True
15331533
Ignore NaN values when inferring the type.

0 commit comments

Comments
 (0)