Xr202601 identical fix, take 2 (#197)

* Fixes to dataset equivalence testing on xarray loads.

* Added towncrier fragment.

* Use own concept of 'dataset equivalence'.

* Simplify xr.Dataset equivalence test.

* Reconfigure docs to ref SciTools.

Author: pp-mo

docs/changelog_fragments/195.dev.rst

@@ -0,0 +1 @@
+Fixed xarray load tests for new behaviour of xarray.Dataset.identical.

docs/conf.py

@@ -19,8 +19,8 @@
 # -- Project information -----------------------------------------------------
 
 project = "ncdata"
-copyright = "2023, pp-mo"
-author = "pp-mo"
+copyright = "2023, SciTools"
+author = "SciTools"
 
 # The complete version, including alpha/beta/rc tags
 version_parts = [str(part) for part in version_tuple]
@@ -149,7 +149,7 @@
 html_context = {
     # Possibly needed for pydata_theme?
     "github_repo": "ncdata",
-    "github_user": "pp-mo",
+    "github_user": "SciTools",
     "github_version": "main",
     "doc_path": "docs",
     # Default light/dark mode.

pyproject.toml

@@ -11,7 +11,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "ncdata"
 authors = [
-    {name = "Patrick Peglar", email = "patrick.peglar@metoffice.gov.uk"},
+    {name = "Iris Contributors", email = "scitools.pub@gmail.com"}
 ]
 description = "Abstract NetCDF data objects, providing fast data transfer between analysis packages."
 requires-python = ">=3.10"
@@ -50,10 +50,10 @@ dependencies = ["numpy", "dask", "netCDF4"]
 
 
 [project.urls]
-Code = "https://github.com/pp-mo/ncdata"
-Discussions = "https://github.com/pp-mo/ncdata/discussions"
+Code = "https://github.com/SciTools/ncdata"
+Discussions = "https://github.com/SciTools/ncdata/discussions"
 Documentation = "https://ncdata.readthedocs.io"
-Issues = "https://github.com/pp-mo/ncdata/issues"
+Issues = "https://github.com/SciTools/ncdata/issues"
 
 [tool.setuptools]
 license-files = ["LICENSE"]
@@ -88,7 +88,7 @@ package = "ncdata"
 package_dir = "lib"
 directory = "docs/changelog_fragments"
 filename = "docs/change_log.rst"
-issue_format = "`ISSUE#{issue} <https://github.com/pp-mo/ncdata/pull/{issue}>`_"
+issue_format = "`ISSUE#{issue} <https://github.com/SciTools/ncdata/pull/{issue}>`_"
 underlines = ["~", "^", "*", "+"]
 
 [[tool.towncrier.type]]

tests/integration/test_xarray_load_and_save_equivalence.py

@@ -6,6 +6,7 @@
 (2) check equivalence of files : xarray -> file VS xarray->ncdata->file
 """
 
+import numpy as np
 import pytest
 import xarray
 
@@ -38,6 +39,45 @@ def use_xarraylock():
         yield
 
 
+def check_load_equivalence(ds1: xarray.Dataset, ds2: xarray.Dataset):
+    """
+    Check that datasets differ only in "expected" ways.
+
+    The key differences are due to coordinates remaining lazy in loading via ncdata, but
+    having real data in a "normal" load.  This also affects which coords have indexes,
+    but we are not checking that here anyway.
+    """
+
+    def check_attrs_equivalent(attrs1, attrs2):
+        # Because dict-eq does not work when values can be arrays (!)
+        okay = set(attrs1.keys()) == set(attrs2.keys())
+        if okay:
+            for attr in attrs1:
+                okay = np.all(attrs1[attr] == attrs2[attr])
+                if not okay:
+                    break
+        assert okay
+
+    def check_vars_equivalent(v1, v2):
+        check_attrs_equivalent(v1.attrs, v2.attrs)
+        assert v1.dims == v2.dims
+        assert v1.dtype == v2.dtype
+        # Numeric compare may need to allow for NaNs : floats *and datetimes*
+        equal_nan = (
+            v1.dtype.kind in "fM"
+        )  # cannot set kwarg when not applicable
+        result = np.array_equal(v1.data, v2.data, equal_nan=equal_nan)
+        if hasattr(result, "compute"):
+            result = result.compute()
+        assert result
+
+    check_attrs_equivalent(ds1.attrs, ds2.attrs)
+    assert ds1.dims == ds2.dims
+    assert list(ds1.variables) == list(ds2.variables)
+    for varname in ds1.variables:
+        check_vars_equivalent(ds1.variables[varname], ds2.variables[varname])
+
+
 def test_load_direct_vs_viancdata(standard_testcase, use_xarraylock, tmp_path):
     source_filepath = standard_testcase.filepath
     ncdata = from_nc4(source_filepath)
@@ -52,8 +92,7 @@ def test_load_direct_vs_viancdata(standard_testcase, use_xarraylock, tmp_path):
     # Load same, via ncdata
     xr_ncdata_ds = to_xarray(ncdata)
 
-    # Treat as OK if it passes xarray comparison
-    assert xr_ds.identical(xr_ncdata_ds)
+    check_load_equivalence(xr_ds, xr_ncdata_ds)
 
 
 def test_save_direct_vs_viancdata(standard_testcase, tmp_path):