Add support for decimal logical type (#86)

* IN PROGRESS
Added in types and fields definitions for Decimal. Added in tests for fields. NEED MORE TESTS AT MIN.

* Changed field logic to error out on no or bad default, restricted fake to match schema, updated tests

* Added comments to utils

* Shifted decimal serialization functions from utils to serialization

* Added schema and serialization tests

* Updated docs

* Linted, tested, and cleaned up

* Added in # type: ignore, Checker was complaining about
dataclasses_avroschema/fields.py:816: error: Incompatible types in assignment (expression has type "type", variable has type "Type[InmutableField]")
dataclasses_avroschema/fields.py:816: note: "type.__call__" has type "Callable[[VarArg(Any), KwArg(Any)], Any]"

and couldn't figure out why

* Upping code coverage

* Cleaned up code per review, added in minor validation checks

Author: xgamer4

.gitignore

@@ -79,6 +79,8 @@ coverage
 
 # Workspace files are user-specific
 *.sublime-workspace
+# Removes PyCharm/JetBrains workspace files
+.idea/
 
 # Project files should be checked into the repository, unless a significant
 # proportion of contributors will probably not be using Sublime Text

dataclasses_avroschema/fields.py

@@ -2,6 +2,7 @@
 import collections
 import dataclasses
 import datetime
+import decimal
 import json
 import random
 import typing
@@ -12,7 +13,7 @@
 from faker import Faker
 from pytz import utc
 
-from dataclasses_avroschema import types, utils
+from dataclasses_avroschema import serialization, types, utils
 
 fake = Faker()
 p = inflect.engine()
@@ -33,6 +34,7 @@
 TIME_MILLIS = "time-millis"
 TIMESTAMP_MILLIS = "timestamp-millis"
 UUID = "uuid"
+DECIMAL = "decimal"
 LOGICAL_DATE = {"type": INT, "logicalType": DATE}
 LOGICAL_TIME = {"type": INT, "logicalType": TIME_MILLIS}
 LOGICAL_DATETIME = {"type": LONG, "logicalType": TIMESTAMP_MILLIS}
@@ -61,20 +63,25 @@
 
 PYTHON_PRIMITIVE_CONTAINERS = (list, tuple, dict)
 
-PYTHON_LOGICAL_TYPES = (
-    datetime.date,
-    datetime.time,
-    datetime.datetime,
-    uuid.uuid4,
-    uuid.UUID,
-)
+PYTHON_LOGICAL_TYPES = (datetime.date, datetime.time, datetime.datetime, uuid.uuid4, uuid.UUID, decimal.Decimal)
 
 PYTHON_PRIMITIVE_TYPES = PYTHON_INMUTABLE_TYPES + PYTHON_PRIMITIVE_CONTAINERS
 
 PRIMITIVE_AND_LOGICAL_TYPES = PYTHON_INMUTABLE_TYPES + PYTHON_LOGICAL_TYPES
 
 PythonImnutableTypes = typing.Union[
-    str, int, bool, float, list, tuple, dict, datetime.date, datetime.time, datetime.datetime, uuid.UUID
+    str,
+    int,
+    bool,
+    float,
+    list,
+    tuple,
+    dict,
+    datetime.date,
+    datetime.time,
+    datetime.datetime,
+    uuid.UUID,
+    decimal.Decimal,
 ]
 
 
@@ -638,6 +645,71 @@ def fake(self) -> typing.Any:
         return self.type.fake()
 
 
+@dataclasses.dataclass
+class DecimalField(BaseField):
+
+    precision: int = -1
+    scale: int = 0
+
+    def __post_init__(self) -> None:
+        self.set_precision_scale()
+
+    def set_precision_scale(self) -> None:
+        if self.default != types.MissingSentinel:
+            if isinstance(self.default, decimal.Decimal):
+                sign, digits, scale = self.default.as_tuple()
+                self.scale = scale * -1  # Make scale positive, as that's what Avro expects
+                # decimal.Context has a precision property
+                # BUT the precision property is independent of the number of digits stored in the Decimal instance
+                # # # FROM THE DOCS HERE https://docs.python.org/3/library/decimal.html
+                #  The context precision does not affect how many digits are stored.
+                #  That is determined exclusively by the number of digits in value.
+                #  For example, Decimal('3.00000') records all five zeros even if the context precision is only three.
+                # # #
+                # Avro is concerned with *what form the number takes* and not with *handling errors in the Python env*
+                # so we take the number of digits stored in the decimal as Avro precision
+                self.precision = len(digits)
+            elif isinstance(self.default, types.Decimal):
+                self.scale = self.default.scale
+                self.precision = self.default.precision
+            else:
+                raise ValueError("decimal.Decimal default types must be either decimal.Decimal or types.Decimal")
+        else:
+            raise ValueError(
+                "decimal.Decimal default types must be specified to provide precision and scale,"
+                " and must be either decimal.Decimal or types.Decimal"
+            )
+
+        # Validation on precision and scale per Avro schema
+        if self.precision <= 0:
+            raise ValueError("Precision must be a positive integer greater than zero")
+
+        if self.scale < 0 or self.precision < self.scale:
+            raise ValueError("Scale must be zero or a positive integer less than or equal to the precision.")
+
+            # Just pull the precision from default context and default out scale
+            # Not ideal
+            #
+            # self.precision = decimal.Context().prec
+
+    def get_avro_type(self) -> typing.Dict[str, typing.Any]:
+        avro_type = {"type": BYTES, "logicalType": DECIMAL, "precision": self.precision, "scale": self.scale}
+
+        return avro_type
+
+    def get_default_value(self) -> typing.Union[str, dataclasses._MISSING_TYPE, None]:
+        default = self.default
+        if isinstance(default, types.Decimal):
+            default = default.default
+
+        if default == types.MissingSentinel:
+            return dataclasses.MISSING
+        return serialization.decimal_to_str(default, self.precision, self.scale)
+
+    def fake(self) -> decimal.Decimal:
+        return fake.pydecimal(right_digits=self.scale, left_digits=self.precision - self.scale)
+
+
 INMUTABLE_FIELDS_CLASSES = {
     bool: BooleanField,
     int: LongField,
@@ -665,6 +737,7 @@ def fake(self) -> typing.Any:
     uuid.uuid4: UUIDField,
     uuid.UUID: UUIDField,
     bytes: BytesField,
+    decimal.Decimal: DecimalField,
 }
 
 PRIMITIVE_LOGICAL_TYPES_FIELDS_CLASSES = {
@@ -742,7 +815,7 @@ def field_factory(
             default_factory=default_factory,
         )
     elif native_type in PYTHON_LOGICAL_TYPES:
-        klass = LOGICAL_TYPES_FIELDS_CLASSES[native_type]
+        klass = LOGICAL_TYPES_FIELDS_CLASSES[native_type]  # type: ignore
         return klass(name=name, type=native_type, default=default, metadata=metadata)
     else:
         return RecordField(name=name, type=native_type, default=default, metadata=metadata)

dataclasses_avroschema/serialization.py

@@ -1,4 +1,5 @@
 import datetime
+import decimal
 import io
 import typing
 import uuid
@@ -61,6 +62,41 @@ def time_to_str(value: datetime.time) -> str:
     return value.strftime(TIME_STR_FORMAT)
 
 
+def decimal_to_str(value: decimal.Decimal, precision: int, scale: int = 0) -> str:
+    value_bytes = prepare_bytes_decimal(value, precision, scale)
+    return r"\u" + value_bytes.hex()
+
+
+# This is an almost complete copy of fastavro's _logical_writers_py.prepare_bytes_decimal
+# the only tweak is to pass in scale/precision directly instead of a schema
+# This is needed to properly serialize a default decimal.Decimal into the avro schema
+def prepare_bytes_decimal(data: decimal.Decimal, precision: int, scale: int = 0) -> bytes:
+    """Convert decimal.Decimal to bytes"""
+
+    sign, digits, exp = data.as_tuple()
+
+    if len(digits) > precision:
+        raise ValueError("The decimal precision is bigger than allowed by schema")
+
+    delta = exp + scale
+
+    if delta < 0:
+        raise ValueError("Scale provided in schema does not match the decimal")
+
+    unscaled_datum = 0
+    for digit in digits:
+        unscaled_datum = (unscaled_datum * 10) + digit
+
+    unscaled_datum = 10 ** delta * unscaled_datum
+
+    bytes_req = (unscaled_datum.bit_length() + 8) // 8
+
+    if sign:
+        unscaled_datum = -unscaled_datum
+
+    return unscaled_datum.to_bytes(bytes_req, byteorder="big", signed=True)
+
+
 def to_json(data: typing.Dict[str, typing.Any]) -> typing.Dict:
     json_data = {}
 
@@ -73,7 +109,7 @@ def to_json(data: typing.Dict[str, typing.Any]) -> typing.Dict:
             value = date_to_str(value)
         elif isinstance(value, datetime.time):
             value = time_to_str(value)
-        elif isinstance(value, uuid.UUID):
+        elif isinstance(value, (uuid.UUID, decimal.Decimal)):
             value = str(value)
 
         json_data[field] = value

dataclasses_avroschema/types.py

@@ -49,4 +49,25 @@ def __repr__(self) -> str:
         return f"{self.symbols}"
 
 
-CUSTOM_TYPES = ("Fixed", "Enum")
+@dataclasses.dataclass
+class Decimal(typing.Generic[T]):
+    """
+    Represents an Avro Decimal type
+
+    precision (int): Specifying the number precision
+    scale(int): Specifying the number scale. Default 0
+    """
+
+    precision: int
+    scale: int = 0
+    default: typing.Any = dataclasses.field(default=MissingSentinel)
+    _dataclasses_custom_type: str = "Decimal"
+
+    # Decimal serializes to bytes, which doesn't support namespace
+    aliases: typing.Optional[typing.List] = None
+
+    def __repr__(self) -> str:
+        return f"Decimal precision: {self.precision} scale:{self.scale}"
+
+
+CUSTOM_TYPES = ("Fixed", "Enum", "Decimal")

docs/fields_specification.md

@@ -81,13 +81,16 @@ Language implementations must ignore unknown logical types when reading, and sho
 
 * UUID: Represents a uuid as a string
 
+* Decimal: Represents a decimal.Decimal as bytes
+
 | Avro Type | Logical Type |Python Type |
 |-----------|--------------|-------------|
 | int       |  date        | datetime.date
 | int       |  time-millis | datetime.time     |
 | long      |  timestamp-millis | datetime.datetime |
 | string    |  uuid        | uuid.uuid4 |
 | string    |  uuid        | uuid.UUID |
+| bytes     | decimal      | decimal.Decimal |
 
 ### Avro Field and Python Types Summary
 
@@ -115,6 +118,7 @@ Python Type | Avro Type   | Logical Type |
 | datetime.time | int     |  time-millis |
 | datetime.datetim| long  |  timestamp-millis |
 | uuid.uuid4  | string    |  uuid        |
+| decimal.Decimal | bytes | decimal      |
 
 ## Adding Custom Field-level Attributes
 

docs/logical_types.md

@@ -9,6 +9,7 @@ The following list represent the avro logical types mapped to python types:
 | long      |  timestamp-millis | datetime.datetime |
 | string    |  uuid        | uuid.uuid4 |
 | string    |  uuid        | uuid.UUID |
+| bytes     | decimal      | decimal.Decimal
 
 ### Date
 
@@ -205,3 +206,76 @@ UUIDLogicalTypes.avro_schema()
   "doc": "UUID logical types"
 }'
 ```
+
+### Decimal
+
+The below code shows an example on how to use decimals. There's a few important things to note:
+* A default IS REQUIRED in order to set scale and precision on the Avro schema
+* It is strongly recommended to set these explicitly using `types.Decimal(scale=, precision=)`
+* They can be set implicitly by using a default `decimal.Decimal`
+* If set implicitly, scale and precision will be derived from the default as follows:
+```python
+    default: decimal.Decimal = decimal.Decimal('3.14')
+    sign, digits, exp = default.as_tuple()
+    precision = len(digits)
+    scale = exp * -1  # Avro schema defines scale as a positive, as_tuple provides negative
+```
+* This CAN and WILL have strange consequences if not careful, ESPECIALLY if constructing `decimal.Decimal` with a float. For example:
+```python
+    string_definition: decimal.Decimal = decimal.Decimal('3.14')
+    # scale = 2, precision = 3
+    float_definition: decimal.Decimal = decimal.Decimal(3.14)
+    # scale = 51, precision = 52
+```
+
+```python
+import decimal
+
+from dataclasses_avroschema import AvroModel, types
+
+
+class DecimalLogicalTypes(AvroModel):
+    "Decimal logical types"
+    explicit: decimal.Decimal = types.Decimal(scale=2, precision=3)
+    explicit_with_default: decimal.Decimal = types.Decimal(scale=2, precision=3, default=decimal.Decimal('3.14'))
+    implicit: decimal.Decimal = decimal.Decimal('3.14') # sets scale = 2, precision = 3, derived from provided default
+    # will_error: decimal.Decimal  # THIS WILL ERROR
+DecimalLogicalTypes.avro_schema()
+
+'{
+  "type": "record",
+  "name": "DecimalLogicalTypes",
+  "fields": [
+    {
+      "name": "explicit",
+      "type": {
+        "type": "bytes",
+        "logicalType": "decimal",
+        "precision": 3,
+        "scale": 2
+      }
+    },
+    {
+      "name": "explicit_with_default",
+      "type": {
+        "type": "bytes",
+        "logicalType": "decimal",
+        "precision": 3,
+        "scale": 2
+      },
+      "default": "\\u013a"
+    },
+    {
+      "name": "implicit",
+      "type": {
+        "type": "bytes",
+        "logicalType": "decimal",
+        "precision": 3,
+        "scale": 2
+      },
+      "default": "\\u013a"
+    }
+  ],
+  "doc": "Decimal logical types"
+}'
+```

tests/fake/test_fake.py

@@ -1,9 +1,10 @@
 import dataclasses
 import datetime
+import decimal
 import typing
 import uuid
 
-from dataclasses_avroschema import AvroModel
+from dataclasses_avroschema import AvroModel, types
 
 
 def test_fake_primitive_types(user_dataclass):
@@ -131,3 +132,17 @@ class User(AvroModel):
         teamates: typing.Dict[str, typing.Type["User"]] = None
 
     assert isinstance(User.fake(), User)
+
+
+def test_decimals():
+    """
+    Test Decimal logical types
+    """
+
+    class User(AvroModel):
+        name: str
+        age: int
+        test_score_1: decimal.Decimal = decimal.Decimal("100.00")
+        test_score_2: decimal.Decimal = types.Decimal(scale=5, precision=11)
+
+    assert isinstance(User.fake(), User)