.github/workflows/benchmarks.yml

@@ -10,7 +10,7 @@ jobs:
     # if: ${{ contains( github.event.pull_request.labels.*.name, 'run-benchmark') && github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}  # Run if the PR has been labelled correctly.
     if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} # Always run.
     name: Linux
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       ASV_DIR: "./asv_bench"
 

.github/workflows/ci-additional.yaml

@@ -77,7 +77,7 @@ jobs:
           --ignore flox/tests \
           --cov=./ --cov-report=xml
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v5.4.0
+        uses: codecov/codecov-action@v5.4.2
         with:
           file: ./coverage.xml
           flags: unittests
@@ -132,7 +132,7 @@ jobs:
           python -m mypy --install-types --non-interactive --cache-dir=.mypy_cache/ --cobertura-xml-report mypy_report
 
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v5.4.0
+        uses: codecov/codecov-action@v5.4.2
         with:
           file: mypy_report/cobertura.xml
           flags: mypy

.github/workflows/ci.yaml

@@ -76,7 +76,7 @@ jobs:
           python -c "import xarray; xarray.show_versions()"
           pytest --durations=20 --durations-min=0.5 -n auto --cov=./ --cov-report=xml --hypothesis-profile ci
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v5.4.0
+        uses: codecov/codecov-action@v5.4.2
         with:
           file: ./coverage.xml
           flags: unittests

.github/workflows/upstream-dev-ci.yaml

@@ -78,7 +78,8 @@ jobs:
             git+https://github.com/Unidata/cftime
           python -m pip install \
             git+https://github.com/dask/dask \
-            git+https://github.com/ml31415/numpy-groupies
+            git+https://github.com/ml31415/numpy-groupies \
+            git+https://github.com/pydata/sparse
 
       - name: Install flox
         run: |

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ ci:
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: "v0.9.1"
+    rev: "v0.11.4"
     hooks:
       - id: ruff
         args: ["--fix", "--show-fixes"]
@@ -24,7 +24,7 @@ repos:
       - id: check-docstring-first
 
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.21
+    rev: 0.7.22
     hooks:
       - id: mdformat
         additional_dependencies:
@@ -38,19 +38,19 @@ repos:
         args: [--extra-keys=metadata.kernelspec metadata.language_info.version]
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.3.0
+    rev: v2.4.1
     hooks:
       - id: codespell
         additional_dependencies:
           - tomli
 
   - repo: https://github.com/abravalheri/validate-pyproject
-    rev: v0.23
+    rev: v0.24.1
     hooks:
       - id: validate-pyproject
 
   - repo: https://github.com/rhysd/actionlint
-    rev: v1.7.6
+    rev: v1.7.7
     hooks:
       - id: actionlint
         files: ".github/workflows/"

asv_bench/benchmarks/combine.py

@@ -14,7 +14,11 @@ def _get_combine(combine):
     if combine == "grouped":
         return partial(flox.core._grouped_combine, engine="numpy")
     else:
-        return partial(flox.core._simple_combine, reindex=False)
+        try:
+            reindex = flox.ReindexStrategy(blockwise=False)
+        except AttributeError:
+            reindex = False
+        return partial(flox.core._simple_combine, reindex=reindex)
 
 
 class Combine:
@@ -41,7 +45,7 @@ def peakmem_combine(self, kind, combine):
 class Combine1d(Combine):
     """
     Time the combine step for dask reductions,
-    this is for reducting along a single dimension
+    this is for reducing along a single dimension
     """
 
     def setup(self, *args, **kwargs) -> None:

ci/docs.yml

@@ -15,6 +15,7 @@ dependencies:
   - matplotlib-base
   - myst-parser
   - myst-nb
+  - sparse
   - sphinx
   - sphinx-remove-toctrees
   - furo>=2024.08

ci/env-numpy1.yml

@@ -11,6 +11,7 @@ dependencies:
   - pandas
   - numpy<2
   - scipy
+  - sparse
   - lxml # for mypy coverage report
   - matplotlib
   - pip

ci/environment.yml

@@ -11,6 +11,7 @@ dependencies:
   - pandas
   - numpy>=1.22
   - scipy
+  - sparse
   - lxml # for mypy coverage report
   - matplotlib
   - pip

ci/no-dask.yml

@@ -8,6 +8,7 @@ dependencies:
   - cftime
   - numpy>=1.22
   - scipy
+  - sparse
   - pip
   - pytest
   - pytest-cov

ci/no-numba.yml

@@ -11,6 +11,7 @@ dependencies:
   - pandas
   - numpy>=1.22
   - scipy
+  - sparse
   - lxml # for mypy coverage report
   - matplotlib
   - pip

ci/no-xarray.yml

@@ -7,6 +7,7 @@ dependencies:
   - pandas
   - numpy>=1.22
   - scipy
+  - sparse
   - pip
   - pytest
   - pytest-cov

docs/source/arrays.md

@@ -1,5 +1,13 @@
 # Duck Array Support
 
+## Sparse Arrays
+
+`sparse.COO` arrays from the `pydata/sparse` project are supported using algorithms that work on the underlying dense data.
+See `aggregate_sparse.py` for details.
+At the moment the following reductions are supported: `sum`, `nansum`, `min`, `nanmin`, `max`, `nanmax`, `count`.
+
+## Other array types
+
 Aggregating over other array types will work if the array types supports the following methods, [ufunc.reduceat](https://numpy.org/doc/stable/reference/generated/numpy.ufunc.reduceat.html) or [ufunc.at](https://numpy.org/doc/stable/reference/generated/numpy.ufunc.at.html)
 
 | Reduction                      | `method="numpy"` | `method="flox"`   |

docs/source/implementation.md

@@ -110,6 +110,26 @@ width: 100%
 
 This approach allows grouping by a dask array so group labels can be discovered at compute time, similar to `dask.dataframe.groupby`.
 
+### reindexing to a sparse array
+
+For large numbers of groups, we might be reducing to a very sparse array (e.g. [this issue](https://github.com/xarray-contrib/flox/issues/428)).
+
+To control memory, we can instruct flox to reindex the intermediate results to a `sparse.COO` array using:
+
+```python
+from flox import ReindexArrayType, ReindexStrategy
+
+ReindexStrategy(
+    # do not reindex to the full output grid at the blockwise aggregation stage
+    blockwise=False,
+    # when combining intermediate results after blockwise aggregation, reindex to the
+    # common grid using a sparse.COO array type
+    array_type=ReindexArrayType.SPARSE_COO,
+)
+```
+
+See [this user story](user-stories/large-zonal-stats) for more discussion.
+
 ### Example
 
 For example, consider `groupby("time.month")` with monthly frequency data and chunksize of 4 along `time`.

docs/source/user-stories.md

@@ -10,4 +10,5 @@
    user-stories/climatology-hourly-cubed.ipynb
    user-stories/custom-aggregations.ipynb
    user-stories/nD-bins.ipynb
+   user-stories/large-zonal-stats.ipynb
 ```