Merge pull request #929 from dcs4cop/forman-928-ref_data_store

Add `reference` data store

Author: TonioF

CHANGES.md

@@ -2,15 +2,23 @@
 
 ### Enhancements
 
-* STAC
+* Added new `reference` filesystem data store to support 
+  "kerchunked" NetCDF files in object storage. (#928)
+  
+  See also
+    - [ReferenceFileSystem](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.implementations.reference.ReferenceFileSystem)
+    - [kerchunk](https://github.com/fsspec/kerchunk)
+
+* Improved xcube Server's STAC API:
   * Provide links for multiple coverages data formats
   * Add `crs` and `crs_storage` properties to STAC data
   * Add spatial and temporal grid data to collection descriptions
   * Add a schema endpoint returning a JSON schema of a dataset's data
     variables
   * Add links to domain set, range type, and range schema to collection
     descriptions
-* OGC API – Coverages:
+
+* Improved xcube Server's Coverages API:
   * Support scaling parameters `scale-factor`, `scale-axes`, and `scale-size`
   * Improve handling of bbox parameters
   * Handle half-open datetime intervals
@@ -27,9 +35,11 @@
 
 ### Fixes
 
-* Make S3 unit tests compatible with moto 5 server (#922)
-* Make some CLI unit tests compatible with pytest 8 (#922)
-* Rename some test classes to avoid spurious warnings (#924)
+* Fixed `KeyError: 'lon_bnds'` raised occasionally when opening 
+  (mostly NetCDF) datasets. (#930)
+* Make S3 unit tests compatible with moto 5 server. (#922)
+* Make some CLI unit tests compatible with pytest 8. (#922)
+* Rename some test classes to avoid spurious warnings. (#924)
 
 ### Other changes
 

test/core/store/fs/test_registry.py

@@ -1,10 +1,10 @@
+from abc import ABC, abstractmethod
 import collections.abc
 import os.path
 import shutil
+from typing import Any, Callable, Dict, Optional, Set, Type, Union
 import unittest
 import warnings
-from abc import ABC, abstractmethod
-from typing import Any, Callable, Dict, Optional, Set, Type, Union
 
 import fsspec
 import numpy as np
@@ -342,7 +342,7 @@ def _assert_dataset_supported(
             requested_dtype_alias: Optional[str],
             expected_dtype_aliases: Set[str],
             expected_return_type: Union[Type[xr.Dataset],
-                                        Type[MultiLevelDataset]],
+            Type[MultiLevelDataset]],
             expected_descriptor_type: Optional[Union[
                 Type[DatasetDescriptor],
                 Type[MultiLevelDatasetDescriptor]

test/core/store/ref/__init__.py

@@ -0,0 +1,20 @@
+# The MIT License (MIT)
+# Copyright (c) 2020-2024 by the xcube development team and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.

test/core/store/ref/test_schema.py

@@ -0,0 +1,54 @@
+# The MIT License (MIT)
+# Copyright (c) 2020-2024 by the xcube development team and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import unittest
+import json
+from xcube.core.store.ref.schema import REF_STORE_SCHEMA
+from xcube.util.jsonschema import JsonObjectSchema
+
+
+class ReferenceSchemaTest(unittest.TestCase):
+
+    def test_schema_schema(self):
+        self.assertIsInstance(REF_STORE_SCHEMA, JsonObjectSchema)
+        self.assertIsInstance(REF_STORE_SCHEMA.properties, dict)
+        self.assertEqual(
+            {
+                'asynchronous',
+                'cache_size',
+                'listings_expiry_time',
+                'max_block',
+                'max_gap',
+                'max_paths',
+                'refs',
+                'remote_options',
+                'remote_protocol',
+                'skip_instance_cache',
+                'target_options',
+                'target_protocol',
+                'use_listings_cache'
+            },
+            set(REF_STORE_SCHEMA.properties.keys())
+        )
+
+    def test_json_serialisation(self):
+        d = REF_STORE_SCHEMA.to_dict()
+        self.assertEqual(d, json.loads(json.dumps(d)))