Add support for PEP585 & PEP604 type defs (#154)

Unpin fastjsonschema
Remove mypy_extensions

Author: s-knibbs

dataclasses_jsonschema/__init__.py

@@ -4,7 +4,7 @@
 import functools
 from decimal import Decimal
 from ipaddress import IPv4Address, IPv6Address
-from typing import Optional, Type, Union, Any, Dict, Tuple, List, TypeVar, get_type_hints, Callable
+from typing import Optional, Type, Union, Any, Dict, Tuple, List, Callable, TypeVar
 import re
 from datetime import datetime, date
 from dataclasses import fields, is_dataclass, Field, MISSING, dataclass, asdict
@@ -18,11 +18,12 @@
     from typing_extensions import Final, Literal  # type: ignore
 
 
-from dataclasses_jsonschema.field_types import (  # noqa: F401
+from .field_types import (  # noqa: F401
     FieldEncoder, DateFieldEncoder, DateTimeFieldEncoder, UuidField, DecimalField,
-    IPv4AddressField, IPv6AddressField, DateTimeField
+    IPv4AddressField, IPv6AddressField, DateTimeField, UUID_REGEX
 )
-from dataclasses_jsonschema.type_defs import JsonDict, SchemaType, JsonSchemaMeta, _NULL_TYPE, NULL  # noqa: F401
+from .type_defs import JsonDict, SchemaType, JsonSchemaMeta, _NULL_TYPE, NULL  # noqa: F401
+from .type_hints import get_class_type_hints
 
 try:
     import fastjsonschema
@@ -46,8 +47,12 @@ def validate_func(data, schema):
 SEQUENCE_TYPES = {
     'Sequence': list,
     'List': list,
-    'Set': set
+    'Set': set,
+    'set': set,
+    'list': list
 }
+MAPPING_TYPES = ('Dict', 'Mapping', 'dict')
+TUPLE_TYPES = ('Tuple', 'tuple')
 
 
 class ValidationError(Exception):
@@ -310,15 +315,15 @@ def encoder(_, v, __):
                 if encoded is None:
                     raise TypeError("No variant of '{}' matched the type '{}'".format(field_type, type(value)))
                 return encoded
-            elif field_type_name in ('Mapping', 'Dict'):
+            elif field_type_name in MAPPING_TYPES:
                 def encoder(ft, val, o):
                     return {
                         cls._encode_field(ft.__args__[0], k, o): cls._encode_field(ft.__args__[1], v, o)
                         for k, v in val.items()
                     }
-            elif field_type_name in SEQUENCE_TYPES or (field_type_name == "Tuple" and ... in field_type.__args__):
+            elif field_type_name in SEQUENCE_TYPES or (field_type_name in TUPLE_TYPES and ... in field_type.__args__):
                 def encoder(ft, val, o): return [cls._encode_field(ft.__args__[0], v, o) for v in val]
-            elif field_type_name == 'Tuple':
+            elif field_type_name in TUPLE_TYPES:
                 def encoder(ft, val, o):
                     return [cls._encode_field(ft.__args__[idx], v, o) for idx, v in enumerate(val)]
             elif cls._is_json_schema_subclass(field_type):
@@ -344,7 +349,7 @@ def _get_fields_uncached():
                 base_fields_types |= {(f.name, f.type) for f in fields(base)}
 
             mapped_fields = []
-            type_hints = get_type_hints(cls)
+            type_hints = get_class_type_hints(cls)
             for f in fields(cls):
                 # Skip internal fields
                 if f.name.startswith("__") or (not base_fields and (f.name, f.type) in base_fields_types):
@@ -433,18 +438,18 @@ def decoder(f, ft, val): return cls._decode_field(f, unwrap_final(ft), val)
                         continue
                 if decoded is not None:
                     return decoded
-            elif field_type_name in ('Mapping', 'Dict'):
+            elif field_type_name in MAPPING_TYPES:
                 def decoder(f, ft, val):
                     return {
                         cls._decode_field(f, ft.__args__[0], k): cls._decode_field(f, ft.__args__[1], v)
                         for k, v in val.items()
                     }
-            elif field_type_name in SEQUENCE_TYPES or (field_type_name == "Tuple" and ... in field_type.__args__):
-                seq_type = tuple if field_type_name == "Tuple" else SEQUENCE_TYPES[field_type_name]
+            elif field_type_name in SEQUENCE_TYPES or (field_type_name in TUPLE_TYPES and ... in field_type.__args__):
+                seq_type = tuple if field_type_name in TUPLE_TYPES else SEQUENCE_TYPES[field_type_name]
 
                 def decoder(f, ft, val):
                     return seq_type(cls._decode_field(f, ft.__args__[0], v) for v in val)
-            elif field_type_name == "Tuple":
+            elif field_type_name in TUPLE_TYPES:
                 def decoder(f, ft, val):
                     return tuple(cls._decode_field(f, ft.__args__[idx], v) for idx, v in enumerate(val))
             elif field_type in cls._field_encoders:
@@ -469,7 +474,9 @@ def _validate(cls, data: JsonDict, validate_enums: bool = True):
                 # TODO: Support validating with other schema types
                 schema_validator = cls.__compiled_schema.get(SchemaOptions(DEFAULT_SCHEMA_TYPE, validate_enums))
                 if schema_validator is None:
-                    schema_validator = fastjsonschema.compile(cls.json_schema(validate_enums=validate_enums))
+                    schema_validator = fastjsonschema.compile(
+                        cls.json_schema(validate_enums=validate_enums), formats={'uuid': UUID_REGEX}
+                    )
                     cls.__compiled_schema[SchemaOptions(DEFAULT_SCHEMA_TYPE, validate_enums)] = schema_validator
                 schema_validator(data)
             else:
@@ -545,7 +552,7 @@ def from_object(cls: Type[T], obj: Any, exclude: FieldExcludeList = tuple()) ->
                 values[f.field.name] = ft.from_object(from_value, exclude=sub_exclude)
             elif is_enum(ft):
                 values[f.field.name] = ft(from_value)
-            elif field_type_name == "List" and cls._is_json_schema_subclass(ft.__args__[0]):
+            elif field_type_name in ("List", "list") and cls._is_json_schema_subclass(ft.__args__[0]):
                 values[f.field.name] = [
                     ft.__args__[0].from_object(v, exclude=sub_exclude) for v in from_value
                 ]
@@ -653,19 +660,19 @@ def _get_field_schema(cls, field: Union[Field, Type], schema_options: SchemaOpti
                 field_schema = {
                     'oneOf': [cls._get_field_schema(variant, schema_options)[0] for variant in field_type.__args__]
                 }
-            elif field_type_name in ('Dict', 'Mapping'):
+            elif field_type_name in MAPPING_TYPES:
                 field_schema = {'type': 'object'}
                 if field_type.__args__[1] is not Any:
                     field_schema['additionalProperties'] = cls._get_field_schema(
                         field_type.__args__[1], schema_options
                     )[0]
-            elif field_type_name in SEQUENCE_TYPES or (field_type_name == "Tuple" and ... in field_type.__args__):
+            elif field_type_name in SEQUENCE_TYPES or (field_type_name in TUPLE_TYPES and ... in field_type.__args__):
                 field_schema = {'type': 'array'}
                 if field_type.__args__[0] is not Any:
                     field_schema['items'] = cls._get_field_schema(field_type.__args__[0], schema_options)[0]
-                if field_type_name == "Set":
+                if field_type_name in ("Set", "set"):
                     field_schema['uniqueItems'] = True
-            elif field_type_name == "Tuple":
+            elif field_type_name in TUPLE_TYPES:
                 tuple_len = len(field_type.__args__)
                 # TODO: How do we handle Optional type within lists / tuples
                 field_schema = {
@@ -691,9 +698,9 @@ def _get_field_definitions(cls, field_type: Any, definitions: JsonDict,
         field_type_name = cls._get_field_type_name(field_type)
         if is_optional(field_type):
             cls._get_field_definitions(unwrap_optional(field_type), definitions, schema_options)
-        elif field_type_name in ('Sequence', 'List', 'Tuple'):
+        elif field_type_name in SEQUENCE_TYPES:
             cls._get_field_definitions(field_type.__args__[0], definitions, schema_options)
-        elif field_type_name in ('Dict', 'Mapping'):
+        elif field_type_name in MAPPING_TYPES:
             cls._get_field_definitions(field_type.__args__[1], definitions, schema_options)
         elif field_type_name == 'Union':
             for variant in field_type.__args__:
@@ -760,7 +767,7 @@ def json_schema(
         else:
             definitions = cls.__definitions[schema_options]
 
-        if cls.__schema is not None and schema_options in cls.__schema:
+        if schema_options in cls.__schema:
             schema = cls.__schema[schema_options]
         else:
             properties = {}

dataclasses_jsonschema/field_types.py

@@ -63,6 +63,7 @@ def json_schema(self) -> JsonDict:
 
 # Alias for backwards compat
 DateTimeField = DateTimeFieldEncoder
+UUID_REGEX = '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
 
 
 class UuidField(FieldEncoder[UUID, str]):
@@ -78,7 +79,7 @@ def json_schema(self):
         return {
             'type': 'string',
             'format': 'uuid',
-            'pattern': '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+            'pattern': UUID_REGEX
         }
 
 

dataclasses_jsonschema/type_defs.py

@@ -5,14 +5,15 @@
     # Supported in future python versions
     from typing import TypedDict  # type: ignore
 except ImportError:
-    from mypy_extensions import TypedDict
+    from typing_extensions import TypedDict
 
 
 JsonEncodable = Union[int, float, str, bool]
 JsonDict = Dict[str, Any]
 
 
-class JsonSchemaMeta(TypedDict):
+# This issue still seems to be present for python < 3.8: https://github.com/python/mypy/issues/7722
+class JsonSchemaMeta(TypedDict, total=False):  # type: ignore
     """JSON schema field definitions. Example usage:
 
     >>> foo = field(metadata=JsonSchemaMeta(description="A foo that foos"))

dataclasses_jsonschema/type_hints.py

@@ -0,0 +1,102 @@
+import ast
+import sys
+
+try:
+    from typing import ForwardRef  # type: ignore
+except ImportError:
+    # Python 3.6
+    from typing import _ForwardRef as ForwardRef  # type: ignore
+
+from typing import Type, Dict, Any, Union, Tuple, Set, List
+from typing import _eval_type  # type: ignore
+
+
+def get_elts(op: ast.BinOp):
+    for arg in (op.left, op.right):
+        if isinstance(arg, ast.BinOp) and isinstance(arg.op, ast.BitOr):
+            for n in get_elts(arg):
+                yield n
+        else:
+            yield arg
+
+
+class RewriteUnionTypes(ast.NodeTransformer):
+
+    def __init__(self):
+        self.rewritten = False
+
+    def visit_BinOp(self, node: ast.BinOp) -> Union[ast.BinOp, ast.Subscript]:
+        if isinstance(node.op, ast.BitOr):
+            self.rewritten = True
+            return ast.Subscript(
+                value=ast.Name(id='_Union', ctx=ast.Load(), lineno=1, col_offset=1),
+                slice=ast.Index(
+                    value=ast.Tuple(elts=list(get_elts(node)), ctx=ast.Load(), lineno=1, col_offset=1),
+                    ctx=ast.Load(),
+                    lineno=1,
+                    col_offset=1,
+                ),
+                lineno=1,
+                col_offset=1,
+                ctx=ast.Load(),
+            )
+        else:
+            return node
+
+
+class RewriteBuiltinGenerics(ast.NodeTransformer):
+
+    def __init__(self):
+        self.rewritten = False
+        # Collections are prefixed with _ to prevent any potential name clashes
+        self.replacements = {
+            'list': '_List',
+            'set': '_Set',
+            'frozenset': '_Frozenset',
+            'dict': '_Dict',
+            'type': '_Type',
+            'tuple': '_Tuple',
+        }
+
+    def visit_Name(self, node: ast.Name) -> ast.Name:
+        if node.id in self.replacements:
+            self.rewritten = True
+            return ast.Name(id=self.replacements[node.id], ctx=node.ctx, lineno=1, col_offset=1)
+        else:
+            return node
+
+
+def get_class_type_hints(klass: Type, localns=None) -> Dict[str, Any]:
+    """Return type hints for a class. Adapted from `typing.get_type_hints`, adds support for PEP 585 & PEP 604"""
+    hints = {}
+    for base in reversed(klass.__mro__):
+        base_globals = sys.modules[base.__module__].__dict__
+        base_globals['_Union'] = Union
+        if sys.version_info < (3, 9):
+            base_globals['_List'] = List
+            base_globals['_Set'] = Set
+            base_globals['_Type'] = Type
+            base_globals['_Tuple'] = Tuple
+            base_globals['_Dict'] = Dict
+        ann = base.__dict__.get('__annotations__', {})
+        for name, value in ann.items():
+            if value is None:
+                value = type(None)
+            if isinstance(value, str):
+                t = ast.parse(value, '<unknown>', 'eval')
+                union_transformer = RewriteUnionTypes()
+                t = union_transformer.visit(t)
+                builtin_generics_transformer = RewriteBuiltinGenerics()
+                if sys.version_info < (3, 9):
+                    t = builtin_generics_transformer.visit(t)
+                if builtin_generics_transformer.rewritten or union_transformer.rewritten:
+                    # Note: ForwardRef raises a TypeError when given anything that isn't a string, so we need
+                    # to compile & eval the ast here
+                    code = compile(t, '<unknown>', 'eval')
+                    hints[name] = eval(code, base_globals, localns)
+                    continue
+                else:
+                    value = ForwardRef(value, is_argument=False)
+            value = _eval_type(value, base_globals, localns)
+            hints[name] = value
+    return hints

setup.py

@@ -3,7 +3,6 @@
 requires = [
     'python-dateutil',
     'jsonschema',
-    'mypy_extensions',
     'typing_extensions;python_version<"3.8"',
     'dataclasses;python_version<"3.7"'
 ]
@@ -24,7 +23,7 @@ def read(f):
     url='https://github.com/s-knibbs/dataclasses-jsonschema',
     install_requires=requires,
     extras_require={
-        'fast-validation': ["fastjsonschema==2.13"],
+        'fast-validation': ["fastjsonschema"],
         'apispec': ["apispec"]
     },
     setup_requires=['pytest-runner', 'setuptools_scm'],

tests/test_peps.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from typing import List
+
+from dataclasses_jsonschema import JsonSchemaMixin
+
+
+def test_pep_604_types():
+    @dataclass
+    class Post(JsonSchemaMixin):
+        body: str
+        tags: str | List[str] | None
+        metadata: List[str | int]
+
+    schema = Post.json_schema()
+    assert schema['properties']['tags'] == {
+        'oneOf': [{'type': 'string'}, {'type': 'array', 'items': {'type': 'string'}}]
+    }
+    assert schema['properties']['metadata'] == {
+        'type': 'array', 'items': {'oneOf': [{'type': 'string'}, {'type': 'integer'}]}
+    }
+    assert schema['required'] == ['body', 'metadata']
+
+
+def test_pep_585_types():
+
+    @dataclass
+    class Collections(JsonSchemaMixin):
+        a: list[str]
+        b: dict[str, int]
+        c: set[int]
+        d: tuple[int, str]
+
+    schema = Collections.json_schema()
+    assert schema['properties'] == {
+        'a': {'type': 'array', 'items': {'type': 'string'}},
+        'b': {'additionalProperties': {'type': 'integer'}, 'type': 'object'},
+        'c': {'type': 'array', 'items': {'type': 'integer'}, 'uniqueItems': True},
+        'd': {'type': 'array', 'items': [{'type': 'integer'}, {'type': 'string'}], 'maxItems': 2, 'minItems': 2}
+    }
+    assert Collections(a=['foo'], b={'bar': 123}, c={4, 5}, d=(6, 'baz')).to_dict() == {
+        'a': ['foo'], 'b': {'bar': 123}, 'c': [4, 5], 'd': [6, 'baz']
+    }

tox.ini

@@ -22,4 +22,17 @@ deps =
     apispec[yaml]
     apispec_webframeworks
     flask
-    fastvalidation: fastjsonschema==2.13
+    fastvalidation: fastjsonschema
+
+[testenv:py36]
+# Exclude test_peps from py 3.6 since this is not supported
+commands =
+    pytest tests/test_core.py tests/test_apispec_plugin.py
+    flake8 dataclasses_jsonschema
+    mypy dataclasses_jsonschema
+
+[testenv:py36-fastvalidation]
+commands =
+    pytest tests/test_core.py tests/test_apispec_plugin.py
+    flake8 dataclasses_jsonschema
+    mypy dataclasses_jsonschema