datatypes restructured

Author: randriu

umbi/binary/api.py

@@ -2,32 +2,15 @@
 A convenient interface for all (de)serializers in this package.
 """
 
-from typing import Union
-
 from .booleans import *
-from .intervals import *
+from .floats import *
 from .integers import *
+from .intervals import *
 from .rationals import *
 from .strings import *
-from .floats import *
 
-Numeric = Union[int, float]
 
-def standard_value_type_size(value_type: str) -> int:
-    """Return the number of bytes needed to represent a value of the given type."""
-    if "-interval" in value_type:
-        base_value_type = value_type.replace("-interval", "")
-        return 2 * standard_value_type_size(base_value_type)
-    if value_type == "rational":
-        # for rationals, the standard size is 8+8 bytes
-        return standard_value_type_size("int64") + standard_value_type_size("uint64")
-    if value_type == "double":
-        return 8  # size of double
-    _, num_bytes = fixed_size_integer_base_and_size(value_type)
-    return num_bytes
-
-
-def value_to_bytes(value: str | Numeric | Fraction | tuple, value_type: str, little_endian: bool = True) -> bytes:
+def value_to_bytes(value: str | int | float | Fraction | tuple, value_type: str, little_endian: bool = True) -> bytes:
     """
     Convert a value of a given type to a bytestring.
     :param value_type: either string or one of {int32|uint32|int64|uint64|double|rational}[-interval]
@@ -49,7 +32,7 @@ def value_to_bytes(value: str | Numeric | Fraction | tuple, value_type: str, lit
         return fixed_size_integer_to_bytes(value, value_type, little_endian)
 
 
-def bytes_to_value(data: bytes, value_type: str, little_endian: bool = True) -> str | Numeric | Fraction | tuple:
+def bytes_to_value(data: bytes, value_type: str, little_endian: bool = True) -> str | int | float | Fraction | tuple:
     """
     Convert a binary string to a single value of the given type.
     :param value_type: string or one of {int32|uint32|int64|uint64|double|rational}[-interval]
@@ -67,26 +50,22 @@ def bytes_to_value(data: bytes, value_type: str, little_endian: bool = True) ->
         return bytes_to_fixed_size_integer(data, value_type, little_endian)
 
 
-def numeric_pack(value: Numeric, value_type: str, num_bits: int) -> BitArray:
+def numeric_pack(value: int | float, value_type: str, num_bits: int) -> BitArray:
     """Convert a single primitive value of the given type to a fixed-length bit representation."""
     assert value_type in ["int", "uint", "double"], f"unsupported primitive type: {value_type}"
     if value_type == "double":
         assert isinstance(value, float)
         assert num_bits == 64, "double must be represented with 64 bits"
         return double_pack(value)
     assert isinstance(value, int)
-    if value_type == "int":
-        return int_pack(value, num_bits)
-    else:
-        return uint_pack(value, num_bits)
+    signed = value_type == "int"
+    return integer_pack(value, num_bits, signed)
 
 
-def numeric_unpack(bits: BitArray, value_type: str) -> Numeric:
+def numeric_unpack(bits: BitArray, value_type: str) -> int | float:
     """Convert a BitArray to a single primitive value of the given type."""
     assert value_type in ["int", "uint", "double"], f"unsupported primitive type: {value_type}"
     if value_type == "double":
         return double_unpack(bits)
-    elif value_type == "int":
-        return int_unpack(bits)
-    else:
-        return uint_unpack(bits)
+    signed = value_type == "int"
+    return integer_unpack(bits, signed)

umbi/binary/booleans.py

@@ -3,6 +3,7 @@
 """
 
 from typing import Optional
+
 from bitstring import BitArray
 
 

umbi/binary/composites.py

@@ -33,9 +33,7 @@ class Field:
 
     name: str
     type: str
-    size: Optional[int] = (
-        None  # number of bits for values of fixed-size types; None for variable-size types (string, rational)
-    )
+    size: Optional[int] = None # number of bits for values of fixed-size types; None for variable-size types (string, rational)
     lower: Optional[float] = None  # lower bound (for numeric types)
     upper: Optional[float] = None  # upper bound (for numeric types)
     offset: Optional[float] = None  # lower value offset (for numeric types)
@@ -99,39 +97,40 @@ def fields(self):
     def validate(self):
         for item in self._fields:
             item.validate()
-
-    @staticmethod
-    def new_padding(total_bits: int) -> Padding | None:
-        """Create a new padding field to align the total_bits to the next byte boundary."""
-        padding = (8 - total_bits % 8) % 8
-        if padding > 0:
-            return Padding(padding)
-        return None
-
-    def add_paddings(self):
-        """Add padding fields to properly align the composite to byte boundaries."""
-        new_fields = []
-        total_bits = 0
-        for field in self._fields:
-            if isinstance(field, Padding):
-                total_bits += field.padding
-            else:
-                # isinstance(field, Field)
-                if field.type in {"string", "rational"}:
-                    # add padding to next byte boundary
-                    padding = self.new_padding(total_bits)
-                    if padding is not None:
-                        new_fields.append(padding)
-                        total_bits += padding.padding
-                else:
-                    assert field.size is not None
-                    total_bits += field.size
-            new_fields.append(field)
-        # add final padding to byte boundary
-        padding = self.new_padding(total_bits)
-        if padding is not None:
-            new_fields.append(padding)
-        self._fields = new_fields
+        #TODO check alignment
+
+    # @staticmethod
+    # def new_padding(total_bits: int) -> Padding | None:
+    #     """Create a new padding field to align the total_bits to the next byte boundary."""
+    #     padding = (8 - total_bits % 8) % 8
+    #     if padding > 0:
+    #         return Padding(padding)
+    #     return None
+
+    # def add_paddings(self):
+    #     """Add padding fields to properly align the composite to byte boundaries."""
+    #     new_fields = []
+    #     total_bits = 0
+    #     for field in self._fields:
+    #         if isinstance(field, Padding):
+    #             total_bits += field.padding
+    #         else:
+    #             # isinstance(field, Field)
+    #             if field.type in {"string", "rational"}:
+    #                 # add padding to next byte boundary
+    #                 padding = self.new_padding(total_bits)
+    #                 if padding is not None:
+    #                     new_fields.append(padding)
+    #                     total_bits += padding.padding
+    #             else:
+    #                 assert field.size is not None
+    #                 total_bits += field.size
+    #         new_fields.append(field)
+    #     # add final padding to byte boundary
+    #     padding = self.new_padding(total_bits)
+    #     if padding is not None:
+    #         new_fields.append(padding)
+    #     self._fields = new_fields
 
 
 class CompositePacker:
@@ -190,7 +189,7 @@ def pack_fields(self, value_type: CompositeType, values: dict[str, object]) -> b
                 self.add_padding(field.padding)
                 continue
             if field.name not in values:
-                raise ValueError(f"Missing value for field {field.name}")
+                raise ValueError(f"missing value for field {field.name}")
             self.pack_field(field, values[field.name])
         self.assert_buffer_empty()
         return self.bytestring

umbi/binary/floats.py

@@ -1,19 +1,24 @@
 import struct
+
 from bitstring import BitArray
 
+
 def double_to_bytes(value: float, little_endian: bool = True) -> bytes:
     ef = "<" if little_endian else ">"
     return struct.pack(f"{ef}d", value)
 
+
 def bytes_to_double(data: bytes, little_endian: bool = True) -> float:
     ef = "<" if little_endian else ">"
     return struct.unpack(f"{ef}d", data)[0]
 
+
 def double_pack(value: float) -> BitArray:
     """Convert a single double value to a fixed-length bit representation."""
     return BitArray(float=value, length=64)
 
+
 def double_unpack(bits: BitArray) -> float:
     """Convert a fixed-length bit representation to a single double value."""
     assert len(bits) == 64, "double must be represented with 64 bits"
-    return bits.float
+    return bits.float

umbi/binary/integers.py

@@ -5,76 +5,65 @@
 from bitstring import BitArray
 
 
-def check_int_range(value: int, num_bits: int):
-    max_value = (1 << (num_bits - 1)) - 1
-    min_value = -(1 << (num_bits - 1))
-    if not (min_value <= value <= max_value):
-        raise ValueError(f"integer value {value} is out of range for a {num_bits}-bit int [{min_value}, {max_value}]")
-
-def check_uint_range(value: int, num_bits: int):
-    max_value = (1 << num_bits) - 1
-    min_value = 0
+def assert_integer_range(value: int, num_bits: int, signed: bool = True):
+    """Assert that an integer fits in the given number of bits."""
+    if signed:
+        max_value = (1 << (num_bits - 1)) - 1
+        min_value = -(1 << (num_bits - 1))
+    else:
+        max_value = (1 << num_bits) - 1
+        min_value = 0
     if not (min_value <= value <= max_value):
-        raise ValueError(f"integer value {value} is out of range for a {num_bits}-bit uint [{min_value}, {max_value}]")
-
+        raise ValueError(
+            f"integer value {value} is out of range for a {num_bits}-bit {'signed' if signed else 'unsigned'} integer [{min_value}, {max_value}]"
+        )
 
-def int_to_bytes(value: int, num_bytes: int, little_endian: bool = True) -> bytes:
-    check_int_range(value, num_bytes * 8)
-    return value.to_bytes(num_bytes, byteorder="little" if little_endian else "big", signed=True)
 
-def uint_to_bytes(value: int, num_bytes: int, little_endian: bool = True) -> bytes:
-    check_uint_range(value, num_bytes * 8)
-    return value.to_bytes(num_bytes, byteorder="little" if little_endian else "big", signed=False)
+def integer_to_bytes(value: int, num_bytes: int, signed: bool = True, little_endian: bool = True) -> bytes:
+    assert_integer_range(value, num_bytes * 8, signed)
+    return value.to_bytes(num_bytes, byteorder="little" if little_endian else "big", signed=signed)
 
-def bytes_to_int(data: bytes, little_endian: bool = True) -> int:
-    return int.from_bytes(data, byteorder="little" if little_endian else "big", signed=True)
-
-def bytes_to_uint(data: bytes, little_endian: bool = True) -> int:
-    return int.from_bytes(data, byteorder="little" if little_endian else "big", signed=False)
 
+def bytes_to_integer(data: bytes, signed: bool = True, little_endian: bool = True) -> int:
+    return int.from_bytes(data, byteorder="little" if little_endian else "big", signed=signed)
 
 
 def fixed_size_integer_base_and_size(value_type: str) -> tuple[str, int]:
     fixed_size_integers = {"int16", "uint16", "int32", "int64", "uint32", "uint64"}
     if value_type not in fixed_size_integers:
-        raise ValueError(f"value type must be one of {list(fixed_size_integers)} but is {value_type}")
-    if value_type.startswith("int"):
-        base_type = "int"
-    else:
-        base_type = "uint"
+        raise ValueError(f"expected value type one of {list(fixed_size_integers)} but is {value_type}")
+    base_type = "int" if value_type.startswith("int") else "uint"
     num_bytes = int(value_type[-2:]) // 8
     return base_type, num_bytes
 
+
 def fixed_size_integer_to_bytes(value: int, value_type: str, little_endian: bool = True) -> bytes:
-    """Convert a single fixed-size integer value of the given type to a bytestring."""
+    """Convert a fixed-size integer value of the given type to a bytestring."""
     base_type, num_bytes = fixed_size_integer_base_and_size(value_type)
-    method = int_to_bytes if base_type == "int" else uint_to_bytes
-    return method(value, num_bytes, little_endian)
+    return integer_to_bytes(value, num_bytes, signed=(base_type == "int"), little_endian=little_endian)
+
 
 def bytes_to_fixed_size_integer(data: bytes, value_type: str, little_endian: bool = True) -> int:
-    """Convert a binary string to a single fixed-size integer value of the given type."""
+    """Convert a binary string to a fixed-size integer value of the given type."""
     base_type, num_bytes = fixed_size_integer_base_and_size(value_type)
-    assert len(data) == num_bytes, f"data length {len(data)} does not match expected size {num_bytes} for type {value_type}"
-    method = bytes_to_int if base_type == "int" else bytes_to_uint
-    return method(data, little_endian)
-
-
-def int_pack(value: int, num_bits: int) -> BitArray:
-    """Convert a single int value to a fixed-length bit representation."""
-    check_int_range(value, num_bits)
-    return BitArray(int=value, length=num_bits)
+    assert (
+        len(data) == num_bytes
+    ), f"data length {len(data)} does not match expected size {num_bytes} for type {value_type}"
+    return bytes_to_integer(data, signed=(base_type == "int"), little_endian=little_endian)
 
-def uint_pack(value: int, num_bits: int) -> BitArray:
-    """Convert a single uint value to a fixed-length bit representation."""
-    check_uint_range(value, num_bits)
-    return BitArray(uint=value, length=num_bits)
 
+def integer_pack(value: int, num_bits: int, signed: bool = True) -> BitArray:
+    """Convert a single integer value to a fixed-length bit representation."""
+    assert_integer_range(value, num_bits, signed)
+    if signed:
+        return BitArray(int=value, length=num_bits)
+    else:
+        return BitArray(uint=value, length=num_bits)
 
-def int_unpack(bits: BitArray) -> int:
-    """Convert a BitArray to a single int value."""
-    return bits.int
-
-def uint_unpack(bits: BitArray) -> int:
-    """Convert a BitArray to a single uint value."""
-    return bits.uint
 
+def integer_unpack(bits: BitArray, signed: bool = True) -> int:
+    """Convert a BitArray to a single integer value."""
+    if signed:
+        return bits.int
+    else:
+        return bits.uint

umbi/binary/intervals.py

@@ -2,15 +2,20 @@
 
 from fractions import Fraction
 
-from .rationals import rational_size, rational_to_bytes, bytes_to_rational
-from .floats import double_to_bytes, bytes_to_double
+from .floats import bytes_to_double, double_to_bytes
+from .rationals import bytes_to_rational, rational_size, rational_to_bytes
 
+def assert_continuous_datatype(value_type: str):
+    """Assert that the given value type is a continuous datatype (rational or double)."""
+    continuous_datatypes = {"rational", "double"}
+    if value_type not in continuous_datatypes:
+        raise ValueError(f"expected value type one of {list(continuous_datatypes)} but is {value_type}")
 
 def interval_to_bytes(value: tuple[object, object], value_type: str, little_endian: bool = True) -> bytes:
     """Convert a tuple of two integers into a bytestring representing an interval."""
     assert len(value) == 2, "interval value must be a pair"
     base_value_type = value_type.replace("-interval", "")
-    assert base_value_type in {"rational", "double"}, f"unsupported base value type for interval: {base_value_type}"
+    assert_continuous_datatype(base_value_type)
     lower, upper = value
     if base_value_type == "rational":
         assert isinstance(lower, Fraction) and isinstance(upper, Fraction)
@@ -31,7 +36,7 @@ def bytes_to_interval(data: bytes, value_type: str, little_endian: bool = True)
     assert len(data) % 2 == 0, "interval data must have even length"
     mid = len(data) // 2
     base_value_type = value_type.replace("-interval", "")
-    assert base_value_type in {"rational", "double"}, f"unsupported base value type for interval: {base_value_type}"
+    assert_continuous_datatype(base_value_type)
     lower, upper = data[:mid], data[mid:]
     converter = {
         "rational": bytes_to_rational,