Merge pull request #2 from marcosschroh/test-schema-serialization

marcosschroh · web-flow · commit 51330f80413b · 2019-09-02T00:37:45.000+02:00
Test schema serialization
diff --git a/dataclasses_avroschema/fields.py b/dataclasses_avroschema/fields.py
@@ -23,9 +23,9 @@
     float: FLOAT,
     bytes: BYTES,
     str: STRING,
-    list: ARRAY,
-    tuple: ENUM,
-    dict: MAP,
+    list: {"type": ARRAY},
+    tuple: {"type": ENUM},
+    dict: {"type": MAP},
 }
 
 # excluding tuple because is a container
@@ -43,6 +43,7 @@ class Field:
     name: str
     type: typing.Any  # store the python type
     default: typing.Any = dataclasses.MISSING
+    default_factory: typing.Any = None
 
     # for avro array field
     items_type: typing.Any = None
@@ -77,17 +78,26 @@ def __post_init__(self):
 
             self.type = origin
 
-    @property
-    def to_avro_type(self) -> PythonPrimitiveTypes:
-        if self.type in PYTHON_PRIMITIVE_TYPES:
-            avro_type = PYTHON_TYPE_TO_AVRO.get(self.type)
+    def get_avro_type(self) -> PythonPrimitiveTypes:
+        avro_type = PYTHON_TYPE_TO_AVRO.get(self.type)
 
+        if self.type in PYTHON_INMUTABLE_TYPES:
             if self.default is not dataclasses.MISSING and self.type is not tuple:
                 if self.default is not None:
                     return [avro_type, NULL]
                 # means that default value is None
                 return [NULL, avro_type]
 
+            return avro_type
+        elif self.type in PYTHON_PRIMITIVE_CONTAINERS:
+            if self.items_type:
+                avro_type["items"] = self.items_type
+            elif self.values_type:
+                avro_type["values"] = self.values_type
+            elif self.symbols:
+                avro_type["symbols"] = self.symbols
+
+            avro_type["name"] = self.name
             return avro_type
         else:
             # we need to see what to to when is a custom type
@@ -102,15 +112,22 @@ def get_default_value(self):
             elif self.type is list:
                 if self.default is None:
                     return []
-
-                assert isinstance(self.default, list), f"List is required as default for field {self.name}"
-                return self.default
             elif self.type is dict:
                 if self.default is None:
                     return {}
+        elif self.default_factory not in (dataclasses.MISSING, None):
+            if self.type is list:
+                # expeting a callable
+                default = self.default_factory()
+                assert isinstance(default, list), f"List is required as default for field {self.name}"
 
-                assert isinstance(self.default, dict), f"Dict is required as default for field {self.name}"
-                return self.default
+                return default
+            elif self.type is dict:
+                # expeting a callable
+                default = self.default_factory()
+                assert isinstance(default, dict), f"Dict is required as default for field {self.name}"
+
+                return default
 
     def render(self) -> OrderedDict:
         """
@@ -127,26 +144,19 @@ def render(self) -> OrderedDict:
             The default key is optional.
 
             If self.type is:
-                * list, the OrderedDict will contains the key items
-                * tuple, he OrderedDict will contains the key symbols
-                * dict, he OrderedDict will contains the key values
+                * list, the OrderedDict will contains the key items inside type
+                * tuple, he OrderedDict will contains the key symbols inside type
+                * dict, he OrderedDict will contains the key values inside type
         """
         template = OrderedDict([
             ("name", self.name),
-            ("type", self.to_avro_type),
+            ("type", self.get_avro_type()),
         ])
 
         default = self.get_default_value()
         if default is not None:
             template["default"] = default
 
-        if self.items_type:
-            template["items"] = self.items_type
-        elif self.values_type:
-            template["values"] = self.values_type
-        elif self.symbols:
-            template["symbols"] = self.symbols
-
         return template
 
     def to_json(self) -> str:
diff --git a/dataclasses_avroschema/schema_generator.py b/dataclasses_avroschema/schema_generator.py
@@ -22,7 +22,12 @@ def generate_dataclass(klass_or_instance):
 
     def parse_dataclasses_fields(self) -> typing.List[Field]:
         return [
-            Field(dataclass_field.name, dataclass_field.type, dataclass_field.default)
+            Field(
+                dataclass_field.name,
+                dataclass_field.type,
+                dataclass_field.default,
+                dataclass_field.default_factory
+            )
             for dataclass_field in dataclasses.fields(self.dataclass)
         ]
 
@@ -40,7 +45,7 @@ def avro_schema(self) -> str:
         return json.dumps(self.generate_schema())
 
     def to_python(self) -> typing.Dict[str, typing.Any]:
-        return json.loads(self.render())
+        return json.loads(self.avro_schema())
 
     @property
     def get_fields(self) -> typing.List[Field]:
diff --git a/docs/avro_schema.md b/docs/avro_schema.md
@@ -54,10 +54,107 @@ class User:
     has_pets: bool
     money: float
 
-avro_schema = SchemaGenerator(User).avro_schema()
+SchemaGenerator(User).avro_schema()
+
+{
+  "type": "record",
+  "name": "User",
+  "fields": [
+    {
+      "name": "name",
+      "type": "string"
+    }
+    ,
+    {
+      "name": "age",
+      "type": "int"
+    },
+    {
+      "name": "has_pets",
+      "type": "boolean"
+    },
+    {
+      "name": "money",
+      "type": "float"
+    }
+  ],
+  "doc": "User(name: str, age: int, has_pets: bool, money: float)"
+}'
 ```
 
-and that is it!! Each python field is related with a avro type. You can find the field relationships here:
+and that is it!! Each python field is related with a avro type. You can find the field relationships (here)[https://marcosschroh.github.io/dataclasses-avroschema/fields_specification/]:
+
+### Enum, Array and Map fields
+
+```python
+class UserAdvance:
+    name: str
+    age: int
+    pets: typing.List[str] = dataclasses.field(default_factory=lambda: ['dog', 'cat'])  # array field with default
+    accounts: typing.Dict[str, int] = dataclasses.field(default_factory=lambda: {"key": 1})  # map field with default
+    has_car: bool = False
+    favorite_colors: typing.Tuple[str] = ("BLUE", "YELLOW", "GREEN")  # enum field
+    country: str = "Argentina"
+    address: str = None
+
+SchemaGenerator(UserAdvance, include_schema_doc=False).avro_schema()
+
+'{
+  "type": "record",
+  "name": "UserAdvance",
+  "fields": [
+    {
+      "name": "name",
+      "type": "string"
+    },
+    {
+      "name": "age",
+      "type": "int"
+    },
+    {
+      "name": "pets",
+      "type": {
+        "type": "array",
+        "items": "string",
+        "name": "pets"
+      },
+      "default": ["dog", "cat"]
+    },
+    {
+      "name": "accounts",
+      "type": {
+        "type": "map",
+        "values": "int",
+        "name": "accounts"
+      },
+      "default": {"key": 1}
+    },
+    {
+      "name": "has_car",
+      "type": ["boolean", "null"],
+      "default": false
+    },
+    {
+      "name": "favorite_colors",
+      "type": {
+        "type": "enum",
+        "symbols": ["BLUE", "YELLOW", "GREEN"],
+        "name": "favorite_colors"
+      }
+    },
+    {
+      "name": "country",
+      "type": ["string", "null"],
+      "default": "Argentina"
+    },
+    {
+      "name": "address",
+      "type": ["null", "string"],
+      "default": "null"
+    }
+  ]
+}'
+```
 
 ### Special Avro attributes
 
diff --git a/docs/fields_specification.md b/docs/fields_specification.md
@@ -26,7 +26,6 @@ So, the previous types can be matched to:
 | float     |     float   |
 | double    |     float   |
 | null      |     None    |
-| double    |     wip     |
 | bytes     |     wip     |
 
 Example:
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,5 @@
+fastavro>=0.22.3
+
 # Code quality
 # ------------------------------------------------------------------------------
 black==19.3b0
diff --git a/setup.py b/setup.py
@@ -5,7 +5,7 @@
 
 from setuptools import setup, find_packages
 
-__version__ = "0.1.0"
+__version__ = "0.1.1"
 
 with open("README.md") as readme_file:
     long_description = readme_file.read()
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,11 +1,7 @@
 import os
-
 import json
-
 import dataclasses
-
 import typing
-
 import pytest
 
 
@@ -78,6 +74,21 @@ class UserAdvance:
     return UserAdvance
 
 
+@pytest.fixture
+def user_advance_with_defaults_dataclass():
+    class UserAdvance:
+        name: str
+        age: int
+        pets: typing.List[str] = dataclasses.field(default_factory=lambda: ['dog', 'cat'])
+        accounts: typing.Dict[str, int] = dataclasses.field(default_factory=lambda: {"key": 1})
+        has_car: bool = False
+        favorite_colors: typing.Tuple[str] = ("BLUE", "YELLOW", "GREEN")
+        country: str = "Argentina"
+        address: str = None
+
+    return UserAdvance
+
+
 @pytest.fixture
 def user_avro_json():
     user_avro_path = os.path.join(AVRO_SCHEMAS_DIR, "user.avsc")
@@ -99,24 +110,15 @@ def user_advance_avro_json():
     return json.loads(schema_string)
 
 
+@pytest.fixture
+def user_advance_with_defaults_avro_json():
+    user_avro_path = os.path.join(AVRO_SCHEMAS_DIR, "user_advance_with_defaults.avsc")
+    schema_string = load(user_avro_path)
+    return json.loads(schema_string)
+
+
 @pytest.fixture
 def user_extra_avro_attributes():
     user_avro_path = os.path.join(AVRO_SCHEMAS_DIR, "user_extra_avro_attributes.avsc")
     schema_string = load(user_avro_path)
     return json.loads(schema_string)
-
-#  private static final Logger log = LoggerFactory.getLogger(Main.class);
-
-#     public class ProductAccumulator {
-#         private String omnitureId;
-#         private String variantId;
-#         private Double cev;
-#         private Long total;
-
-#         public ProductAccumulator(String omnitureId, String variantId, Double cev, Long total) {
-#             this.omnitureId = omnitureId;
-#             this.variantId = variantId;
-#             this.cev = cev;
-#             this.total = total;
-#         }
-#     }
diff --git a/tests/schemas/avro/user_advance.avsc b/tests/schemas/avro/user_advance.avsc
@@ -2,13 +2,52 @@
   "type": "record",
   "name": "UserAdvance",
   "fields": [
-    {"name": "name", "type": "string"},
-    {"name": "age", "type": "int"},
-    {"name": "pets", "type": "array", "items": "string"},
-    {"name": "accounts", "type": "map", "values": "int"},
-    {"name": "has_car", "type": ["boolean", "null"], "default": false},
-    {"name": "favorite_colors", "type": "enum", "symbols": ["BLUE", "YELLOW", "GREEN"]},
-    {"name": "country", "type": ["string", "null"], "default": "Argentina"},
-    {"name": "address", "type": ["null", "string"], "default": "null"}
+    {
+      "name": "name",
+      "type": "string"
+    },
+    {
+      "name": "age",
+      "type": "int"
+    },
+    {
+      "name": "pets",
+      "type": {
+        "type": "array",
+        "items": "string",
+        "name": "pets"
+      }
+    },
+    {
+      "name": "accounts",
+      "type": {
+        "type": "map",
+        "values": "int",
+        "name": "accounts"
+      }
+    },
+    {
+      "name": "has_car",
+      "type": ["boolean", "null"],
+      "default": false
+    },
+    {
+      "name": "favorite_colors",
+      "type": {
+        "type": "enum",
+        "symbols": ["BLUE", "YELLOW", "GREEN"],
+        "name": "favorite_colors"
+      }
+    },
+    {
+      "name": "country",
+      "type": ["string", "null"],
+      "default": "Argentina"
+    },
+    {
+      "name": "address",
+      "type": ["null", "string"],
+      "default": "null"
+    }
   ]
 }
diff --git a/tests/schemas/avro/user_advance_with_defaults.avsc b/tests/schemas/avro/user_advance_with_defaults.avsc
diff --git a/tests/test_fields.py b/tests/test_fields.py
diff --git a/tests/test_schema.py b/tests/test_schema.py
diff --git a/tests/test_schema_parse.py b/tests/test_schema_parse.py

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+fastavro>=0.22.3`
	`2`	`+`
`1`	`3`	`# Code quality`
`2`	`4`	`# ------------------------------------------------------------------------------`
`3`	`5`	`black==19.3b0`