Merge pull request #3767 from neutrinoceros/stability_backports

REL: manual backports to yt-4.0.x for stability

Author: matthewturk

conftest.py

@@ -4,8 +4,10 @@
 from importlib.util import find_spec
 from pathlib import Path
 
+import matplotlib
 import pytest
 import yaml
+from packaging.version import Version
 
 from yt.config import ytcfg
 from yt.utilities.answer_testing.testing_utilities import (
@@ -17,6 +19,8 @@
     data_dir_load,
 )
 
+MPL_VERSION = Version(matplotlib.__version__)
+
 
 def pytest_addoption(parser):
     """
@@ -109,38 +113,31 @@ def pytest_configure(config):
     ):
         config.addinivalue_line("filterwarnings", value)
 
-    # at the time of writing, astropy's wheels are behind numpy's latest
-    # version but this doesn't cause actual problems in our test suite, so
-    # we allow this warning to pass.
-    # last checked with astropy 4.2.1
-    config.addinivalue_line(
-        "filterwarnings",
-        (
-            "ignore:numpy.ndarray size changed, may indicate binary incompatibility. "
-            "Expected 80 from C header, got 88 from PyObject:RuntimeWarning"
-        ),
-    )
-    if find_spec("astropy") is not None:
-        # astropy triggers this warning from itself, there's not much we can do on our side
-        # last checked with astropy 4.2.1
+    if MPL_VERSION < Version("3.0.0"):
         config.addinivalue_line(
-            "filterwarnings", "ignore::astropy.wcs.wcs.FITSFixedWarning"
+            "filterwarnings",
+            (
+                "ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' "
+                "is deprecated since Python 3.3,and in 3.9 it will stop working:DeprecationWarning"
+            ),
         )
 
-    if find_spec("cartopy") is not None:
-        # cartopy still triggers this numpy warning
-        # last checked wtih cartopy 0.19.0
+    if find_spec("astropy") is not None:
+        # at the time of writing, astropy's wheels are behind numpy's latest
+        # version but this doesn't cause actual problems in our test suite
+        # last updated with astropy 5.0 + numpy 1.22 + pytest 6.2.5
         config.addinivalue_line(
             "filterwarnings",
             (
-                "ignore:`np.float` is a deprecated alias for the builtin `float`. "
-                "To silence this warning, use `float` by itself. "
-                "Doing this will not modify any behavior and is safe. "
-                "If you specifically wanted the numpy scalar type, use `np.float64` here."
-                ":DeprecationWarning: "
+                "ignore:numpy.ndarray size changed, may indicate binary incompatibility. Expected "
+                r"(80 from C header, got 88|88 from C header, got 96|80 from C header, got 96)"
+                " from PyObject:RuntimeWarning"
             ),
         )
-        # this warning *still* shows up on cartopy 0.19 so we'll ignore it
+
+    if find_spec("cartopy") is not None:
+        # This can be removed when cartopy 0.21 is released
+        # see https://github.com/SciTools/cartopy/pull/1957
         config.addinivalue_line(
             "filterwarnings",
             (
@@ -158,6 +155,17 @@ def pytest_configure(config):
             ),
         )
 
+    if find_spec("xarray") is not None:
+        # this can be removed when upstream issue is closed and a fix published
+        # https://github.com/pydata/xarray/issues/6092
+        config.addinivalue_line(
+            "filterwarnings",
+            (
+                "ignore:distutils Version classes are deprecated. "
+                "Use packaging.version instead.:DeprecationWarning"
+            ),
+        )
+
 
 def pytest_collection_modifyitems(config, items):
     r"""

setup.cfg

@@ -92,6 +92,7 @@ full =
     requests>=2.20.0
     scipy>=1.5.0
     xarray>=0.16.1
+    glue-core!=1.2.4;python_version >= '3.10' # see https://github.com/glue-viz/glue/issues/2263
 mapserver =
     bottle
 minimal =

tests/cartopy_requirements.txt

@@ -16,13 +16,7 @@ osx|macOS)
     sudo mkdir -p /usr/local/man
     sudo chown -R "${USER}:admin" /usr/local/man
     brew update
-    # proj can be unpinned when upstream incompatibility issue is resolved. See
-    # https://github.com/SciTools/cartopy/issues/1140
-    export LDFLAGS="$LDFLAGS -L/usr/local/opt/proj@7/lib"
-    export CPPFLAGS="$CPPFLAGS -I/usr/local/opt/proj@7/include"
-    export CFLAGS="$CFLAGS -I/usr/local/opt/proj@7/include"
-    export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/proj@7/lib/pkgconfig"
-    HOMEBREW_NO_AUTO_UPDATE=1 brew install hdf5 proj@7 geos open-mpi netcdf ccache
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install hdf5 proj geos open-mpi netcdf ccache
     ;;
 esac
 
@@ -42,6 +36,18 @@ if [[ "${RUNNER_OS}" == "Windows" ]] && [[ ${dependencies} != "minimal" ]]; then
 else
     python -m pip install --upgrade pip
     python -m pip install --upgrade wheel
+
+    # // band aid
+    # TODO: revert https://github.com/yt-project/yt/pull/3733
+    # when the following upstream PR is released
+    # https://github.com/mpi4py/mpi4py/issues/160
+
+    # workaround taken from
+    # https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
+    export SETUPTOOLS_USE_DISTUTILS=stdlib
+    python -m pip install mpi4py
+    # // end band aid
+
     python -m pip install --upgrade setuptools
 fi
 

tests/ci_install.sh

@@ -1,6 +1,6 @@
 numpy>=1.19.4
 cython>=0.29.21,<3.0
-cartopy~=0.18.0
+cartopy>=0.20.1
 h5py~=3.1.0
 matplotlib!=3.4.2,>=2.1.0  # keep in sync with setup.cfg
 scipy~=1.5.0

tests/windows_conda_requirements.txt

@@ -842,8 +842,8 @@ def compare_image_lists(new_result, old_result, decimals):
     num_images = len(old_result)
     assert num_images > 0
     for i in range(num_images):
-        expected = np.loads(zlib.decompress(old_result[i]))
-        actual = np.loads(zlib.decompress(new_result[i]))
+        expected = pickle.loads(zlib.decompress(old_result[i]))
+        actual = pickle.loads(zlib.decompress(new_result[i]))
         expected_p, actual_p = ensure_image_comparability(expected, actual)
 
         mpimg.imsave(fns[0], expected_p)

yt/utilities/answer_testing/framework.py

@@ -269,7 +269,16 @@ def _init_image(self, data, cbnorm, cblinthresh, cmap, extent, aspect):
             # instances.  It is left as a historical note because we will
             # eventually need some form of it.
             # self.axes.set_extent(extent)
-            pass
+
+            # possibly related issue (operation order dependency)
+            # https://github.com/SciTools/cartopy/issues/1468
+
+            # in cartopy 0.19 (or 0.20), some intented behaviour changes produced an
+            # incompatibility here where default values for extent would lead to a crash.
+            # A solution is to let the transform object set the image extents internally
+            # see https://github.com/SciTools/cartopy/issues/1955
+            extent = None
+
         self.image = self.axes.imshow(
             data.to_ndarray(),
             origin="lower",