fix BYTE_ARRAY_DECIMAL conversion

Author: wolejri

fastparquet/converted_types.py

@@ -178,16 +178,27 @@ def convert(data, se, timestamp96=True, dtype=None):
         if data.dtype.kind in ['i', 'f']:
             return data * scale_factor
         else:  # byte-string
-            # NB: general but slow method
-            # could optimize when data.dtype.itemsize <= 8
-            # TODO: easy cythonize (but rare)
-            # TODO: extension point for pandas-decimal (no conversion needed)
-            return np.array([
-                int.from_bytes(
-                    data.data[i:i + 1], byteorder='big', signed=True
-                ) * scale_factor
-                for i in range(len(data))
-            ])
+            # Handle both FIXED_LEN_BYTE_ARRAY and BYTE_ARRAY
+            if se.type == parquet_thrift.Type.FIXED_LEN_BYTE_ARRAY:
+                # Fixed-length: use buffer slicing approach
+                # This handles the case where numpy may truncate during iteration
+                # (see commit 53ceac2dbb141b76f603318a5cc0f78e64769d62)
+                its = data.dtype.itemsize 
+                by = data.tobytes()
+                return np.array([
+                    int.from_bytes(
+                        by[i * its:(i + 1) * its],
+                        byteorder='big', signed=True
+                    ) * scale_factor
+                    for i in range(len(data))
+                ])
+            else:
+                # Variable-length (BYTE_ARRAY): iterate over elements directly
+                # Each element is a bytes object in the object array
+                return np.array([
+                    int.from_bytes(d, byteorder='big', signed=True) * scale_factor
+                    for d in data
+                ])
     elif ctype == parquet_thrift.ConvertedType.DATE:
         data = data * DAYS_TO_NANOS
         return data.view('datetime64[ns]')

fastparquet/test/test_converted_types.py

@@ -157,6 +157,34 @@ def test_big_decimal():
                       np.array([0., 777.2, 751.6, 345.1, 644.1])).all()
 
 
+def test_byte_array_decimal():
+    """Test decimal data stored as variable-length BYTE_ARRAY."""
+    schema = pt.SchemaElement(
+        type=pt.Type.BYTE_ARRAY,  # Variable length (not FIXED_LEN)
+        name="test",
+        converted_type=pt.ConvertedType.DECIMAL,
+        scale=4,
+        precision=19
+    )
+    # Create DECIMAL values as variable-length byte arrays
+    # Stored as: value * 10^scale
+    # $0.00 -> 0 * 10^4 = 0 -> b'\x00'
+    # $606.00 -> 606 * 10^4 = 6060000 -> b'\x5c\x77\xe0'
+    # $280.00 -> 280 * 10^4 = 2800000 -> b'\x2a\xb9\x80'
+    # $625.00 -> 625 * 10^4 = 6250000 -> b'\x5f\x5e\x10'
+    data = np.array([
+        b'\x00',                    # 0
+        b'\x5c\x77\xe0',           # 6,060,000 (3 bytes)
+        b'\x2a\xb9\x80',           # 2,800,000 (3 bytes)
+        b'\x5f\x5e\x10',           # 6,250,000 (3 bytes)
+    ], dtype=object)
+
+    result = convert(data, schema)
+    expected = np.array([0.0, 606.0, 280.0, 625.0])
+
+    assert np.allclose(result, expected), \
+        f"Expected {expected}, got {result}"
+
 def test_tz_nonstring(tmpdir):
     # https://github.com/dask/fastparquet/issues/578
     import uuid