Merge pull request #589 from spacetelescope/v2.1.x

Release candidate for v2.1.1

Author: drdavella

.travis.yml

@@ -22,7 +22,7 @@ env:
         # overidden underneath. They are defined here in order to save having
         # to repeat them for all configurations.
         - PYTHON_VERSION=3.6
-        - PYTEST_VERSION=3.6
+        - PYTEST_VERSION=3.7
         - ASTROPY_VERSION=stable
         - NUMPY_VERSION=stable
         - PIP_DEPENDENCIES='pytest-faulthandler importlib_resources'
@@ -39,9 +39,10 @@ env:
         - PIP_DEPENDENCIES='collective.checkdocs pygments' SETUP_CMD='checkdocs'
         - PYTHON_VERSION=3.5 SETUP_CMD='test'
         - PYTHON_VERSION=3.6 SETUP_CMD='test'
-        - PYTHON_VERSION=3.7 SETUP_CMD='test'
+        - PYTHON_VERSION=3.7 PYTEST_VERSION=3.8 SETUP_CMD='test'
 
 matrix:
+    fast_finish: true
     include:
 
         # Do a coverage test
@@ -62,6 +63,7 @@ matrix:
         # also test against development version of Astropy
         - env: MAIN_CMD='pytest' SETUP_CMD='' ASTROPY_VERSION=development
                GWCS_PIP="$GWCS_GIT"
+               PYTEST_VERSION=3.8
 
         # latest stable versions
         - env: NUMPY_VERSION=stable SETUP_CMD='test'

CHANGES.rst

@@ -1,3 +1,22 @@
+2.1.1 (2018-11-01)
+------------------
+
+- Make sure extension metadata is written even when constructing the ASDF tree
+  on-the-fly. [#549]
+
+- Fix large integer validation when storing `numpy` integer literals in the
+  tree. [#553]
+
+- Fix bug that caused subclass of external type to be serialized by the wrong
+  tag. [#560]
+
+- Fix bug that occurred when attempting to open invalid file but Astropy import
+  fails while checking for ASDF-in-FITS. [#562]
+
+- Fix bug that caused tree creation to fail when unable to locate a schema file
+  for an unknown tag. This now simply causes a warning, and the offending node
+  is converted to basic Python data structures. [#571]
+
 2.1.0 (2018-09-25)
 ------------------
 

appveyor.yml

@@ -5,6 +5,7 @@ environment:
 
   global:
       PYTHON: "C:\\conda"
+      PYTEST_VERSION: "3.7"
       MINICONDA_VERSION: "latest"
       CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci-helpers\\appveyor\\windows_sdk.cmd"
       SETUP_CMD: "test"
@@ -29,20 +30,22 @@ environment:
         NUMPY_VERSION: "1.12.0"
         platform: x64
 
-      - PYTHON_VERSION: "3.6"
+      - PYTHON_VERSION: "3.7"
+        PYTEST_VERSION: "3.7"
         platform: x64
 
       - PYTHON_VERSION: "3.6"
         ASTROPY_VERSION: "stable"
         platform: x64
 
-      - PYTHON_VERSION: "3.6"
+      - PYTHON_VERSION: "3.7"
+        PYTEST_VERSION: "3.7"
         ASTROPY_VERSION: "development"
         GWCS_PIP: "%GWCS_GIT%"
         platform: x64
 
       # Tests against development version of numpy may fail
-      - PYTHON_VERSION: "3.6"
+      - PYTHON_VERSION: "3.7"
         NUMPY_VERSION: "development"
         platform: x64
 

asdf/asdf.py

@@ -677,10 +677,14 @@ def _open_impl(cls, self, fd, uri=None, mode='r',
                             ignore_missing_extensions=ignore_missing_extensions,
                             _extension_metadata=self._extension_metadata)
             except ValueError:
-                pass
-            raise ValueError(
-                "Input object does not appear to be ASDF file or FITS with " +
-                "ASDF extension")
+                raise ValueError(
+                    "Input object does not appear to be an ASDF file or a FITS with " +
+                    "ASDF extension") from None
+            except ImportError:
+                raise ValueError(
+                    "Input object does not appear to be an ASDF file. Cannot check " +
+                    "if it is a FITS with ASDF extension because 'astropy' is not " +
+                    "installed") from None
         return cls._open_asdf(self, fd, uri=uri, mode=mode,
                 validate_checksums=validate_checksums,
                 do_not_fill_defaults=do_not_fill_defaults,
@@ -827,6 +831,7 @@ def _pre_write(self, fd, all_array_storage, all_array_compression,
         self._blocks.finalize(self)
 
         self._tree['asdf_library'] = get_asdf_library_info()
+        self._update_extension_history()
 
     def _serial_write(self, fd, pad_blocks, include_block_index):
         self._write_tree(self._tree, fd, pad_blocks)
@@ -912,8 +917,6 @@ def update(self, all_array_storage=None, all_array_compression='input',
             write out in the latest version supported by asdf.
         """
 
-        self._update_extension_history()
-
         fd = self._fd
 
         if fd is None:
@@ -922,7 +925,9 @@ def update(self, all_array_storage=None, all_array_compression='input',
 
         if not fd.writable():
             raise IOError(
-                "Can not update, since associated file is read-only")
+                "Can not update, since associated file is read-only. Make "
+                "sure that the AsdfFile was opened with mode='rw' and the "
+                "underlying file handle is writable.")
 
         if version is not None:
             self.version = version
@@ -1054,8 +1059,6 @@ def write_to(self, fd, all_array_storage=None, all_array_compression='input',
             write out in the latest version supported by asdf.
         """
 
-        self._update_extension_history()
-
         original_fd = self._fd
 
         if version is not None:

asdf/asdftypes.py

@@ -126,7 +126,11 @@ def add_subclasses(typ, asdftype):
                 # hierarchy than the existing subclass.
                 if subclass in self._class_by_subclass:
                     if issubclass(self._class_by_subclass[subclass], typ):
-                        continue
+                        # Allow for cases where a subclass tag is being
+                        # overridden by a tag from another extension.
+                        if (self._extension_by_cls[subclass] ==
+                                index._extension_by_type[asdftype]):
+                            continue
                 self._class_by_subclass[subclass] = typ
                 self._type_by_subclasses[subclass] = asdftype
                 self._extension_by_cls[subclass] = index._extension_by_type[asdftype]

asdf/conftest.py

@@ -21,11 +21,6 @@
 TESTED_VERSIONS[packagename] = version.version
 
 
-pytest_plugins = [
-    'asdf.tests.schema_tester'
-]
-
-
 try:
     PYTEST_HEADER_MODULES['Astropy'] = 'astropy'
     PYTEST_HEADER_MODULES['jsonschema'] = 'jsonschema'

asdf/extension.py

@@ -231,4 +231,10 @@ def extension_list(self):
     def package_metadata(self):
         return self._package_metadata
 
+    def reset(self):
+        """This will be used primarily for testing purposes."""
+        self._extensions = []
+        self._extension_list = None
+        self._package_metadata = {}
+
 default_extensions = _DefaultExtensions()

asdf/schema.py

@@ -4,8 +4,11 @@
 import os
 import json
 import datetime
+import warnings
+from numbers import Integral
 from functools import lru_cache
 from collections import OrderedDict
+from collections.abc import Mapping
 from urllib import parse as urlparse
 
 from jsonschema import validators as mvalidators
@@ -217,7 +220,12 @@ def iter_errors(self, instance, _schema=None, _seen=set()):
                 if tag is not None:
                     schema_path = self.ctx.resolver(tag)
                     if schema_path != tag:
-                        s = load_schema(schema_path, self.ctx.resolver)
+                        try:
+                            s = load_schema(schema_path, self.ctx.resolver)
+                        except FileNotFoundError:
+                            msg = "Unable to locate schema file for '{}': '{}'"
+                            warnings.warn(msg.format(tag, schema_path))
+                            s = {}
                         if s:
                             with self.resolver.in_scope(schema_path):
                                 for x in super(ASDFValidator, self).iter_errors(instance, s):
@@ -312,7 +320,6 @@ def load_custom_schema(url):
     core = load_schema(AsdfObject.yaml_tag)
 
     def update(d, u):
-        from collections import Mapping
         for k, v in u.items():
             # Respect the property ordering of the core schema
             if k == 'propertyOrder' and k in d:
@@ -457,7 +464,7 @@ def validate_large_literals(instance):
     """
     # We can count on 52 bits of precision
     for instance in treeutil.iter_tree(instance):
-        if (isinstance(instance, int) and (
+        if (isinstance(instance, (Integral)) and (
             instance > ((1 << 51) - 1) or
             instance < -((1 << 51) - 2))):
             raise ValidationError(

asdf/tags/core/tests/test_history.py

@@ -286,3 +286,12 @@ def types(self):
             pass
 
     assert len(warnings) == 0
+
+    # Make sure that this works even when constructing the tree on-the-fly
+    tmpfile3 = str(tmpdir.join('custom_extension2.asdf'))
+    with asdf.AsdfFile(extensions=FractionExtension()) as ff:
+        ff.tree['fraction'] = fractions.Fraction(4, 5)
+        ff.write_to(tmpfile3)
+
+    with asdf.open(tmpfile3, extensions=FractionExtension()) as af:
+        assert len(af['history']['extensions']) == 2

asdf/tests/__init__.py

@@ -3,6 +3,8 @@
 This packages contains affiliated package tests.
 """
 
+import numpy as np
+
 from .. import CustomType
 
 
@@ -20,3 +22,27 @@ def from_tree_tagged(cls, tree, ctx):
     @classmethod
     def from_tree(cls, tree, ctx):
         return tree
+
+
+def create_small_tree():
+    x = np.arange(0, 10, dtype=np.float)
+    tree = {
+        'science_data': x,
+        'subset': x[3:-3],
+        'skipping': x[::2],
+        'not_shared': np.arange(10, 0, -1, dtype=np.uint8)
+    }
+    return tree
+
+
+def create_large_tree():
+    # These are designed to be big enough so they don't fit in a
+    # single block, but not so big that RAM/disk space for the tests
+    # is enormous.
+    x = np.random.rand(256, 256)
+    y = np.random.rand(16, 16, 16)
+    tree = {
+        'science_data': x,
+        'more': y
+    }
+    return tree