MAINT - Using pixi for nightlies and fixing failing nightly tests (#1664)

Co-authored-by: Riccardo Cappuzzo <riccardo.cappuzzo@gmail.com>
Co-authored-by: Riccardo Cappuzzo <7548232+rcap107@users.noreply.github.com>
Co-authored-by: Gael Varoquaux <gael.varoquaux@normalesup.org>

Author: 3 people

.github/workflows/testing.yml

@@ -15,14 +15,13 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest, ubuntu-latest, macos-latest]
-        environment: [
-            ci-py309-min-deps,
-            ci-py309-min-optional-deps,
-            ci-py311-transformers,
-            ci-py313-latest-deps,
-            ci-py313-latest-optional-deps,
-            ci-py313-polars-without-pyarrow
-        ]
+        environment:
+          - ci-py309-min-deps
+          - ci-py309-min-optional-deps
+          - ci-py311-transformers
+          - ci-py313-latest-deps
+          - ci-py313-latest-optional-deps
+          - ci-py313-polars-without-pyarrow
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v5
@@ -33,13 +32,10 @@ jobs:
           # we can freeze the environment and manually bump the dependencies to the
           # latest version time to time.
           frozen: true
-
       - name: Run tests
         run: pixi run -e ${{ matrix.environment }} test -n auto
-
       - name: Run user guide doctests
         run: pixi run -e ${{ matrix.environment }} test-user-guide
-
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v5.5.1
         with:
@@ -55,28 +51,21 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - id: get_head_commit_message
-        name: get head commit message
+        name: Get head commit message
         run: echo "COMMIT_MSG=$(git show -s --format=%s)" >> "$GITHUB_OUTPUT"
 
   test_against_nightly:
     needs: check_run_nightly
     if: ${{ needs.check_run_nightly.outputs.run == 'true' }}
     runs-on: ubuntu-latest
-    name: test against nighlty dependencies
+    name: Test against nightly dependencies
     steps:
       - uses: actions/checkout@v5
-      - uses: actions/setup-python@v6
+      - uses: prefix-dev/setup-pixi@v0.9.1
         with:
-            python-version: "3.11"
-        name: 'Setup python'
-      - shell: bash {0}
-        run: |
-          dev_anaconda_url=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
-          pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url .
-          dev_arrow=https://pypi.fury.io/arrow-nightlies/
-          pip install --extra-index-url $dev_arrow --prefer-binary --pre .[optional]
-          pip install .[test]
-        name: 'Install skrub and nightly dependencies'
-      - shell: bash {0}
-        run: pytest -vsl --cov=skrub --cov-report=xml skrub -n 3
-        name: 'Run tests'
+          pixi-version: v0.41.3
+          # we can freeze the environment and manually bump the dependencies to the
+          # latest version time to time.
+          frozen: true
+      - name: Run tests
+        run: pixi run --environment ci-nightly-deps pytest -vsl --cov=skrub --cov-report=xml skrub -n 3

CHANGES.rst

@@ -46,7 +46,9 @@ Bugfixes
   columns with some dtypes (lists, structs) are sorted. It would not indicate
   Polars columns sorted in descending order. Fixed in :pr:`1673` by
   :user:`Jérôme Dockès <jeromedockes>`.
-
+- Fixed nightly checks and added support for upcoming library versions, including Pandas
+  v3.0. :pr:`1664` by :user:`Auguste Baum <auguste-probabl>` and
+  :user:`Riccardo Cappuzzo <rcap107>`.
 
 Release 0.6.2
 =============

pixi.lock

@@ -8,11 +8,11 @@ dynamic = ["version"]
 description = "Machine learning with dataframes"
 readme = "README.rst"
 authors = [
-    {name = "Riccardo Cappuzzo"},
-    {name = "Jerome Dockes"},
-    {name = "Guillaume Lemaitre"},
-    {name = "Vincent Maladiere"},
-    {name = "Gael Varoquaux"},
+  { name = "Riccardo Cappuzzo" },
+  { name = "Jerome Dockes" },
+  { name = "Guillaume Lemaitre" },
+  { name = "Vincent Maladiere" },
+  { name = "Gael Varoquaux" },
 ]
 license = "BSD-3-Clause"
 license-files = ["LICENSE.txt"]
@@ -31,56 +31,54 @@ classifiers = [
 ]
 requires-python = ">=3.9"
 dependencies = [
-    "numpy>=1.23.5",
-    "pandas>=1.5.3",
-    "scikit-learn>=1.2.1",
-    "scipy>=1.9.3",
-    "jinja2>=3.1.2",
-    "matplotlib>=3.4.3",
-    "requests>=2.25.0",
-    "pydot",
+  "numpy>=1.23.5",
+  "pandas>=1.5.3",
+  "scikit-learn>=1.2.1",
+  "scipy>=1.9.3",
+  "jinja2>=3.1.2",
+  "matplotlib>=3.4.3",
+  "requests>=2.25.0",
+  "pydot",
 ]
 
 [tool.setuptools.dynamic]
 version = { file = "skrub/VERSION.txt" }
 
 [project.optional-dependencies]
 dev = [
-    "ipykernel",
-    "ipython",
-    "jupyterlab",
-
-    # doc
-    "jupyterlite-sphinx",
-    "jupyterlite-pyodide-kernel",
-    "numpydoc",
-    "pydata-sphinx-theme",
-    "seaborn",
-    "sphinx",
-    "sphinx-copybutton",
-    "sphinx-gallery",
-    "sphinxext-opengraph",
-    "sphinx-autosummary-accessors",
-    "statsmodels",
-
-    # lint
-    "black==23.3.0",
-    "ruff==0.4.8",
-    "pre-commit",
-
-    # test
-    "pytest",
-    "pytest-cov",
-    "pytest-xdist",
-
-    # optional
-    "pyarrow",
-    "polars",
-    "plotly",
-]
-transformers = [
-    "sentence-transformers",
+  "ipykernel",
+  "ipython",
+  "jupyterlab",
+
+  # doc
+  "jupyterlite-sphinx",
+  "jupyterlite-pyodide-kernel",
+  "numpydoc",
+  "pydata-sphinx-theme",
+  "seaborn",
+  "sphinx",
+  "sphinx-copybutton",
+  "sphinx-gallery",
+  "sphinxext-opengraph",
+  "sphinx-autosummary-accessors",
+  "statsmodels",
+
+  # lint
+  "black==23.3.0",
+  "ruff==0.4.8",
+  "pre-commit",
+
+  # test
+  "pytest",
+  "pytest-cov",
+  "pytest-xdist",
+
+  # optional
+  "pyarrow",
+  "polars",
+  "plotly",
 ]
+transformers = ["sentence-transformers"]
 
 [project.urls]
 Homepage = "https://skrub-data.org/"
@@ -90,7 +88,7 @@ Issues = "https://github.com/skrub-data/skrub/issues"
 [tool.setuptools]
 packages = ["skrub"]
 
-[tool.pixi.project]
+[tool.pixi.workspace]
 channels = ["conda-forge", "pytorch"]
 platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"]
 
@@ -169,6 +167,31 @@ pyarrow = "==15.0.2"
 polars = "==0.20.15"
 plotly = "*"
 
+[tool.pixi.feature.nightly-dependencies.pypi-options]
+# Get the nightly of as many dependencies as possible
+extra-index-urls = [
+  # scikit-learn, numpy, pandas...
+  "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple",
+  # pyarrow
+  "https://pypi.fury.io/arrow-nightlies",
+]
+
+[tool.pixi.feature.nightly-dependencies.pypi-dependencies]
+# Have to repeat dependencies here because otherwise pixi pulls from conda channels
+# rather than PyPI indexes
+skrub = { path = ".", editable = true }
+numpy = "*"
+pandas = "*"
+scikit-learn = "*"
+scipy = "*"
+jinja2 = "*"
+matplotlib = "*"
+pydot = "*"
+requests = "*"
+pyarrow = "*"
+polars = "*"
+plotly = "*"
+
 [tool.pixi.feature.polars-without-pyarrow.dependencies]
 polars = "*"
 plotly = "*"
@@ -211,14 +234,22 @@ doc = ["optional", "doc", "transformers"]
 test = ["optional", "test"]
 dev = ["dev", "optional", "doc", "lint", "test", "transformers"]
 ci-py309-min-deps = ["py309", "min-dependencies", "test"]
-ci-py309-min-optional-deps = ["py309", "min-dependencies", "min-optional-dependencies", "test"]
+ci-py309-min-optional-deps = [
+  "py309",
+  "min-dependencies",
+  "min-optional-dependencies",
+  "test",
+]
 ci-py311-transformers = ["py311", "transformers", "test"]
 ci-py312-latest-deps = ["py312", "test"]
 ci-py312-latest-optional-deps = ["py312", "optional", "test"]
 ci-py313-latest-deps = ["py313", "test"]
 ci-py313-polars-without-pyarrow = ["py313", "test", "polars-without-pyarrow"]
 ci-py313-latest-optional-deps = ["py313", "optional", "test"]
 
+# no-default-feature is necessary for pixi to download from nightly PyPI indexes rather than conda channels
+ci-nightly-deps = { no-default-feature = true, features = [ "py313", "nightly-dependencies", "test" ] }
+
 [tool.black]
 line-length = 88
 target_version = ['py310', 'py311']
@@ -241,60 +272,60 @@ exclude = '''
 # max line length for black
 line-length = 88
 target-version = "py38"
-exclude=[
-    ".git",
-    "__pycache__",
-    "dist",
-    "doc/_build",
-    "doc/auto_examples",
-    "build",
+exclude = [
+  ".git",
+  "__pycache__",
+  "dist",
+  "doc/_build",
+  "doc/auto_examples",
+  "build",
 ]
 
 [tool.ruff.lint]
 # all rules can be found here: https://beta.ruff.rs/docs/rules/
 select = ["E", "F", "W", "I"]
-ignore=[
-    # space before : (needed for how black formats slicing)
-    "E203",
-    # do not assign a lambda expression, use a def
-    "E731",
-    # do not use variables named 'l', 'O', or 'I'
-    "E741",
+ignore = [
+  # space before : (needed for how black formats slicing)
+  "E203",
+  # do not assign a lambda expression, use a def
+  "E731",
+  # do not use variables named 'l', 'O', or 'I'
+  "E741",
 ]
 
 [tool.ruff.lint.per-file-ignores]
 # It's fine not to put the import at the top of the file in the examples
 # folder.
-"examples/*"=["E402"]
-"doc/conf.py"=["E402"]
+"examples/*" = ["E402"]
+"doc/conf.py" = ["E402"]
 
 [tool.pytest.ini_options]
 filterwarnings = [
-    # Turn deprecation warnings into errors
-    "error::FutureWarning",
-    "error::DeprecationWarning",
-
-    # Ignore warning from np.in1d since the future behavior is already the desired
-    # behavior. TODO remove when numpy min version >= 1.25.
-    'ignore:elementwise\ comparison\ failed:FutureWarning',
-    # TODO remove when pandas min version >= 2.2.0
-    'ignore:.*utcfromtimestamp\(\) is deprecated.*:DeprecationWarning',
-    # TODO remove when joblib min version >= 1.4.0
-    'ignore:.*ast\.Num is deprecated.*:DeprecationWarning',
-    # TODO remove when joblib min version >= 1.4.0
-    'ignore:.*ast\.NameConstant is deprecated.*:DeprecationWarning',
-    # TODO remove when joblib min version >= 1.4.0
-    'ignore:.*Attribute n is deprecated.*:DeprecationWarning',
-    # FIXME we need to decided what to do with pyarrow that is required by pandas >= 3.0
-    'ignore:(?s).*Pyarrow will become a required dependency of pandas.*:DeprecationWarning',
-    # accessing .values on a pandas dataframe raises this warning after numpy 1.25;
-    # should be addressed in pandas
-    'ignore:np.find_common_type is deprecated.*:DeprecationWarning',
-    'ignore:.*distutils Version classes are deprecated.*',
-    'ignore:.*`resume_download` is deprecated.*',
-    'ignore:.*`clean_up_tokenization_spaces` was not set.*',
-    'ignore:The current process just got forked.*',
-    'ignore:.*`clean_up_tokenization_spaces` was not set.*',
+  # Turn deprecation warnings into errors
+  "error::FutureWarning",
+  "error::DeprecationWarning",
+
+  # Ignore warning from np.in1d since the future behavior is already the desired
+  # behavior. TODO remove when numpy min version >= 1.25.
+  'ignore:elementwise\ comparison\ failed:FutureWarning',
+  # TODO remove when pandas min version >= 2.2.0
+  'ignore:.*utcfromtimestamp\(\) is deprecated.*:DeprecationWarning',
+  # TODO remove when joblib min version >= 1.4.0
+  'ignore:.*ast\.Num is deprecated.*:DeprecationWarning',
+  # TODO remove when joblib min version >= 1.4.0
+  'ignore:.*ast\.NameConstant is deprecated.*:DeprecationWarning',
+  # TODO remove when joblib min version >= 1.4.0
+  'ignore:.*Attribute n is deprecated.*:DeprecationWarning',
+  # FIXME we need to decided what to do with pyarrow that is required by pandas >= 3.0
+  'ignore:(?s).*Pyarrow will become a required dependency of pandas.*:DeprecationWarning',
+  # accessing .values on a pandas dataframe raises this warning after numpy 1.25;
+  # should be addressed in pandas
+  'ignore:np.find_common_type is deprecated.*:DeprecationWarning',
+  'ignore:.*distutils Version classes are deprecated.*',
+  'ignore:.*`resume_download` is deprecated.*',
+  'ignore:.*`clean_up_tokenization_spaces` was not set.*',
+  'ignore:The current process just got forked.*',
+  'ignore:.*`clean_up_tokenization_spaces` was not set.*',
 ]
 addopts = "--doctest-modules"
 doctest_optionflags = "NORMALIZE_WHITESPACE ELLIPSIS"

pyproject.toml

@@ -343,8 +343,8 @@ class ApplyToCols(TransformerMixin, BaseEstimator):
          birthday    city
     0  29/01/2024  London
     >>> df.dtypes
-    birthday    object
-    city        object
+    birthday    ...
+    city        ...
     dtype: object
     >>> ToDatetime().fit_transform(df["birthday"])
     0   2024-01-29
@@ -382,7 +382,7 @@ class ApplyToCols(TransformerMixin, BaseEstimator):
 
     >>> transformed.dtypes
     birthday    datetime64[...]
-    city                object
+    city                ...
     dtype: object
     >>> to_datetime.transformers_
     {'birthday': ToDatetime()}