Migrate to pyproject.toml (#214)

* Migrate to pyproject.toml

* Some linting

* Fix build

* fix version

* fix lint

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* autofix notebooks

* lint more

* Proper docs env install

* Update

* Fix linting

* fix

* Fix

* Fix typing

* Update tests/test_xarray.py

* Cleanup

* Readd setup.py for benchmarks

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: dcherian

.git_archival.txt

@@ -0,0 +1,4 @@
+node: $Format:%H$
+node-date: $Format:%cI$
+describe-name: $Format:%(describe:tags=true)$
+ref-names: $Format:%D$

.github/workflows/testpypi-release.yaml

@@ -0,0 +1,89 @@
+name: Build and Upload to TestPyPI
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+    types: [opened, reopened, synchronize, labeled]
+    branches:
+      - "*"
+  workflow_dispatch:
+
+# no need for concurrency limits
+
+jobs:
+  build-artifacts:
+    if: ${{ contains( github.event.pull_request.labels.*.name, 'test-build') && github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build twine
+          python -m pip install tomli tomli_w
+
+      # - name: Disable local versions
+      #   run: |
+      #     python .github/workflows/configure-testpypi-version.py pyproject.toml
+      #     git update-index --assume-unchanged pyproject.toml
+      #     cat pyproject.toml
+
+      - name: Build tarball and wheels
+        run: |
+          git clean -xdf
+          python -m build
+
+      - name: Check built artifacts
+        run: |
+          python -m twine check --strict dist/*
+          if [ -f dist/flox-999.tar.gz ]; then
+            echo "❌ INVALID VERSION NUMBER"
+            exit 1
+          else
+            echo "✅ Looks good"
+          fi
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: releases
+          path: dist
+
+  test-built-dist:
+    needs: build-artifacts
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: "3.10"
+      - uses: actions/download-artifact@v3
+        with:
+          name: releases
+          path: dist
+      - name: List contents of built dist
+        run: |
+          ls -ltrh
+          ls -ltrh dist
+
+      - name: Verify the built dist/wheel is valid
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install dist/flox*.whl
+          # python -m cf_xarray.scripts.print_versions
+
+      # - name: Publish package to TestPyPI
+      #   uses: pypa/[email protected]
+      #   with:
+      #     password: ${{ secrets.TESTPYPI_TOKEN }}
+      #     repository_url: https://test.pypi.org/legacy/
+      #     verbose: true

.pre-commit-config.yaml

@@ -2,6 +2,13 @@ ci:
     autoupdate_schedule: quarterly
 
 repos:
+    - repo: https://github.com/charliermarsh/ruff-pre-commit
+      # Ruff version.
+      rev: 'v0.0.246'
+      hooks:
+        - id: ruff
+          args: ["--fix"]
+
     - repo: https://github.com/pre-commit/pre-commit-hooks
       rev: v4.4.0
       hooks:
@@ -11,20 +18,10 @@ repos:
         - id: check-docstring-first
 
     - repo: https://github.com/psf/black
-      rev: 22.12.0
+      rev: 23.1.0
       hooks:
         - id: black
 
-    - repo: https://github.com/PyCQA/flake8
-      rev: 6.0.0
-      hooks:
-        - id: flake8
-
-    - repo: https://github.com/PyCQA/isort
-      rev: 5.11.4
-      hooks:
-        - id: isort
-
     - repo: https://github.com/executablebooks/mdformat
       rev: 0.7.16
       hooks:
@@ -37,9 +34,8 @@ repos:
       rev: 1.6.1
       hooks:
           - id: nbqa-black
-          - id: nbqa-pyupgrade
-            args: [--py37-plus]
-          - id: nbqa-isort
+          - id: nbqa-ruff
+            args: [--fix]
 
     - repo: https://github.com/kynan/nbstripout
       rev: 0.6.1
@@ -54,9 +50,7 @@ repos:
           additional_dependencies:
             - tomli
 
-    - repo: https://github.com/asottile/pyupgrade
-      rev: v3.3.1
+    - repo: https://github.com/abravalheri/validate-pyproject
+      rev: v0.12.1
       hooks:
-        - id: pyupgrade
-          args:
-            - "--py38-plus"
+        - id: validate-pyproject

MANIFEST.in

@@ -18,4 +18,4 @@ dependencies:
   - jupyter
   - sphinx-codeautolink
   - pip:
-    - git+https://github.com/xarray-contrib/flox
+    - -e ..

ci/docs.yml

@@ -55,8 +55,6 @@
     }
    ],
    "source": [
-    "import time\n",
-    "\n",
     "import dask.array\n",
     "import numpy as np\n",
     "import pandas as pd\n",

docs/source/user-stories/climatology-hourly.ipynb

@@ -26,7 +26,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import matplotlib.pyplot as plt\n",
     "import numpy as np\n",
     "import numpy_groupies as npg\n",
     "import xarray as xr\n",
@@ -203,7 +202,6 @@
    "outputs": [],
    "source": [
     "def grouped_median(group_idx, array, *, axis=-1, size=None, fill_value=None, dtype=None):\n",
-    "\n",
     "    return npg.aggregate_numpy.aggregate(\n",
     "        group_idx,\n",
     "        array,\n",

docs/source/user-stories/custom-aggregations.ipynb

@@ -5,15 +5,14 @@
 from .aggregations import Aggregation  # noqa
 from .core import groupby_reduce, rechunk_for_blockwise, rechunk_for_cohorts  # noqa
 
-try:
-    from importlib.metadata import version as _version
-except ImportError:
-    # if the fallback library is missing, we are doomed.
-    from importlib_metadata import version as _version  # type: ignore[no-redef]
 
-try:
-    __version__ = _version("flox")
-except Exception:
-    # Local copy or not installed with setuptools.
-    # Disable minimum version checks on downstream libraries.
+def _get_version():
     __version__ = "999"
+    try:
+        from ._version import __version__
+    except ImportError:
+        pass
+    return __version__
+
+
+__version__ = _get_version()

flox/__init__.py

@@ -77,7 +77,6 @@ def _nan_grouped_op(group_idx, array, func, fillna, *args, **kwargs):
 
 
 def sum_of_squares(group_idx, array, *, axis=-1, size=None, fill_value=None, dtype=None):
-
     return sum(
         group_idx,
         array**2,

flox/aggregate_flox.py

@@ -18,7 +18,6 @@ def sum_of_squares(
     fill_value=None,
     dtype=None,
 ):
-
     return _get_aggregate(engine).aggregate(
         group_idx,
         array,
@@ -40,7 +39,6 @@ def nansum_of_squares(
     fill_value=None,
     dtype=None,
 ):
-
     return _get_aggregate(engine).aggregate(
         group_idx,
         array,

flox/aggregate_npg.py

@@ -9,7 +9,8 @@
 import numpy_groupies as npg
 from numpy.typing import DTypeLike
 
-from . import aggregate_flox, aggregate_npg, xrdtypes as dtypes, xrutils
+from . import aggregate_flox, aggregate_npg, xrutils
+from . import xrdtypes as dtypes
 
 if TYPE_CHECKING:
     FuncTuple = tuple[Callable | str, ...]

flox/aggregations.py

@@ -122,7 +122,7 @@ def _get_optimal_chunks_for_groups(chunks, labels):
     firstidx = first_indexes[labels_at_chunk_bounds]
 
     newchunkidx = [0]
-    for c, f, l in zip(chunkidx, firstidx, lastidx):
+    for c, f, l in zip(chunkidx, firstidx, lastidx):  # noqa
         Δf = abs(c - f)
         Δl = abs(c - l)
         if c == 0 or newchunkidx[-1] > l:
@@ -362,7 +362,6 @@ def rechunk_for_blockwise(array: DaskArray, axis: T_Axis, labels: np.ndarray):
 def reindex_(
     array: np.ndarray, from_, to, fill_value=None, axis: T_Axis = -1, promote: bool = False
 ) -> np.ndarray:
-
     if not isinstance(to, pd.Index):
         if promote:
             to = pd.Index(to)
@@ -389,7 +388,7 @@ def reindex_(
         )
     idx = from_.get_indexer(to)
     indexer = [slice(None, None)] * array.ndim
-    indexer[axis] = idx  # type: ignore
+    indexer[axis] = idx
     reindexed = array[tuple(indexer)]
     if any(idx == -1):
         if fill_value is None:
@@ -417,7 +416,7 @@ def offset_labels(labels: np.ndarray, ngroups: int) -> tuple[np.ndarray, int]:
     )
     # -1 indicates NaNs. preserve these otherwise we aggregate in the wrong groups!
     offset[labels == -1] = -1
-    size: int = math.prod(labels.shape[:-1]) * ngroups  # type: ignore
+    size: int = math.prod(labels.shape[:-1]) * ngroups
     return offset, size
 
 
@@ -1213,7 +1212,6 @@ def dask_groupby_agg(
     sort: bool = True,
     chunks_cohorts=None,
 ) -> tuple[DaskArray, tuple[np.ndarray | DaskArray]]:
-
     import dask.array
     from dask.array.core import slices_from_chunks
 
@@ -1556,7 +1554,6 @@ def _factorize_multiple(by, expected_groups, any_by_dask, reindex):
 
 
 def _validate_expected_groups(nby: int, expected_groups: T_ExpectedGroupsOpt) -> T_ExpectTuple:
-
     if expected_groups is None:
         return (None,) * nby
 

flox/core.py

@@ -24,7 +24,6 @@ def draw_mesh(
     y0=0,
     append=False,
 ):
-
     dx = 2
     xpts = x0 + np.arange(0, (ncol + nspaces) * dx, dx)
     ypts = y0 + np.arange(0, nrow * dx, dx)
@@ -118,7 +117,6 @@ def visualize_groups_1d(array, labels, axis=-1, colors=None, cmap=None, append=T
 
 
 def get_colormap(N):
-
     cmap = mpl.cm.get_cmap("tab20_r").copy()
     ncolors = len(cmap.colors)
     q = N // ncolors
@@ -129,7 +127,6 @@ def get_colormap(N):
 
 
 def factorize_cohorts(by, cohorts):
-
     factorized = np.full(by.shape, -1)
     for idx, cohort in enumerate(cohorts):
         factorized[np.isin(by, cohort)] = idx

flox/visualize.py

@@ -15,9 +15,9 @@
     _get_expected_groups,
     _validate_expected_groups,
     groupby_reduce,
-    rechunk_for_blockwise as rechunk_array_for_blockwise,
-    rechunk_for_cohorts as rechunk_array_for_cohorts,
 )
+from .core import rechunk_for_blockwise as rechunk_array_for_blockwise
+from .core import rechunk_for_cohorts as rechunk_array_for_cohorts
 from .xrutils import _contains_cftime_datetimes, _to_pytimedelta, datetime_to_numeric
 
 if TYPE_CHECKING:
@@ -343,7 +343,6 @@ def xarray_reduce(
             raise ValueError("expect_index cannot be None")
 
     def wrapper(array, *by, func, skipna, core_dims, **kwargs):
-
         array, *by = _broadcast_size_one_dims(array, *by, core_dims=core_dims)
 
         # Handle skipna here because I need to know dtype to make a good default choice.
@@ -585,7 +584,6 @@ def resample_reduce(
     keep_attrs: bool = True,
     **kwargs,
 ):
-
     warnings.warn(
         "flox.xarray.resample_reduce is now deprecated. Please use Xarray's resample method directly.",
         DeprecationWarning,

flox/xarray.py

@@ -181,7 +181,6 @@ def datetime_to_numeric(array, offset=None, datetime_unit=None, dtype=float):
 
     # Convert np.NaT to np.nan
     elif array.dtype.kind in "mM":
-
         # Convert to specified timedelta units.
         if datetime_unit:
             array = array / np.timedelta64(1, datetime_unit)