Strip excess segment padding (#8)

* Strip excess segment padding during decoding
* Add support for Python 3.10
* Remove support for Python 3.6

Author: scaramallion

.github/workflows/pytest-builds.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9, "3.10"]
         os: [ubuntu-latest, windows-latest, macos-latest]
 
     steps:

.github/workflows/release-wheels.yml

@@ -9,12 +9,12 @@ jobs:
     name: Build wheels for ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     env:
-      CIBW_SKIP: "cp35-*"
+      CIBW_SKIP: "cp36-*"
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: [3.9]
+        python-version: ["3.10"]
 
     steps:
     - uses: actions/checkout@v2
@@ -35,7 +35,7 @@ jobs:
     - name: Install requirements
       run: |
         pip install -U pip
-        pip install cibuildwheel==1.10.0
+        pip install cibuildwheel==2.3.1
         pip install setuptools-rust
 
     - name: Build sdist

README.md

@@ -1,7 +1,7 @@
 
 ## pylibjpeg-rle
 
-A fast DICOM ([PackBits](https://en.wikipedia.org/wiki/PackBits)) RLE plugin for [pylibjpeg](https://github.com/pydicom/pylibjpeg), written in Rust with a Python 3.6+ wrapper.
+A fast DICOM ([PackBits](https://en.wikipedia.org/wiki/PackBits)) RLE plugin for [pylibjpeg](https://github.com/pydicom/pylibjpeg), written in Rust with a Python 3.7+ wrapper.
 
 Linux, MacOS and Windows are all supported.
 

docs/release_notes/v1.2.0.rst

@@ -0,0 +1,14 @@
+.. _v1.2.0:
+
+1.2.0
+=====
+
+Enhancements
+............
+
+* Support decoding segments with non-conformant padding (:issue:`7`)
+
+Changes
+.......
+
+* Support for Python 3.6 has been dropped, while Python 3.10 is now supported

rle/_version.py

@@ -3,7 +3,7 @@
 import re
 
 
-__version__ = '1.1.0'
+__version__ = '1.2.0'
 
 
 VERSION_PATTERN = r"""

rle/tests/test_decode.py

@@ -136,25 +136,15 @@ def test_invalid_samples_px_raises(self):
         with pytest.raises(ValueError, match=msg):
             decode_frame(d + b'\x00' * 8, 1, 8, '<')
 
-    def test_insufficient_frame_literal_raises(self):
-        """Test exception if frame not large enough to hold segment on lit."""
-        msg = (
-            r"The end of the frame was reached before the segment was "
-            r"completely decoded"
-        )
+    def test_insufficient_frame_literal(self):
+        """Test segment with excess padding on lit."""
         d = self.as_bytes([64])
-        with pytest.raises(ValueError, match=msg):
-            decode_frame(d + b'\x00' * 8, 1, 8, '<')
+        assert decode_frame(d + b'\x00' * 8, 1, 8, '<') == b"\x00"
 
-    def test_insufficient_frame_copy_raises(self):
-        """Test exception if frame not large enough to hold segment on copy."""
-        msg = (
-            r"The end of the frame was reached before the segment was "
-            r"completely decoded"
-        )
+    def test_insufficient_frame_copy(self):
+        """Test segment withe excess padding on copy."""
         d = self.as_bytes([64])
-        with pytest.raises(ValueError, match=msg):
-            decode_frame(d + b'\xff\x00\x00', 1, 8, '<')
+        assert decode_frame(d + b'\xff\x00\x00', 1, 8, '<') == b"\x00"
 
     def test_insufficient_segment_copy_raises(self):
         """Test exception if insufficient segment data on copy."""
@@ -844,3 +834,12 @@ def test_generator(self):
         assert (600, 800) == arr.shape
         with pytest.raises(StopIteration):
             next(gen)
+
+    def test_multi_sample(self):
+        ds = deepcopy(INDEX["SC_rgb_rle_16bit.dcm"]['ds'])
+
+        gen = generate_frames(ds, reshape=True)
+        arr = next(gen)
+        assert (100, 100, 3) == arr.shape
+        with pytest.raises(StopIteration):
+            next(gen)

rle/tests/test_encode.py

@@ -245,8 +245,7 @@ def test_cycle(self, _, ds, nr_frames):
         encoded = encode_frame(ds.PixelData, *params, '<')
         decoded = _rle_decode_frame(encoded, *params)
 
-        dtype = pixel_dtype(ds).newbyteorder('>')
-        arr = np.frombuffer(decoded, dtype)
+        arr = np.frombuffer(decoded, pixel_dtype(ds))
 
         if ds.SamplesPerPixel == 1:
             arr = arr.reshape(ds.Rows, ds.Columns)

rle/tests/test_utils.py

@@ -153,6 +153,13 @@ def test_bad_byteorder_raises(self):
         with pytest.raises(ValueError, match=msg):
             encode_pixel_data(b'', **kwargs)
 
+    def test_encode_using_dataset(self):
+        """Test encoding using a dataset"""
+        ds = INDEX_LEE["SC_rgb_32bit_2frame.dcm"]['ds']
+        src = ds.pixel_array[0].tobytes()
+        enc = encode_pixel_data(src, ds, "<")
+        assert enc[:10] == b"\x0C\x00\x00\x00\x40\x00\x00\x00\x08\x01"
+
     def test_no_byteorder_u8(self):
         """Test exception raised if invalid byteorder."""
         kwargs = {

setup.py

@@ -1,21 +1,15 @@
 
-import os
-import sys
+from pathlib import Path
 from setuptools import setup, find_packages
 from setuptools_rust import Binding, RustExtension
 
-import subprocess
-from distutils.command.build import build as build_orig
-import distutils.sysconfig
 
+VERSION_FILE = Path(__file__).parent / "rle" / '_version.py'
+with open(VERSION_FILE) as f:
+    exec(f.read())
 
-
-VERSION_FILE = os.path.join('rle', '_version.py')
-with open(VERSION_FILE) as fp:
-    exec(fp.read())
-
-with open('README.md', 'r') as fp:
-    long_description = fp.read()
+with open('README.md', 'r') as f:
+    long_description = f.read()
 
 setup(
     name = 'pylibjpeg-rle',
@@ -39,15 +33,13 @@
         "Intended Audience :: Developers",
         "Intended Audience :: Healthcare Industry",
         "Intended Audience :: Science/Research",
-        #"Development Status :: 3 - Alpha",
-        #"Development Status :: 4 - Beta",
         "Development Status :: 5 - Production/Stable",
         "Natural Language :: English",
         "Programming Language :: Rust",
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Operating System :: MacOS :: MacOS X",
         "Operating System :: POSIX :: Linux",
         "Operating System :: Microsoft :: Windows",
@@ -58,7 +50,7 @@
     package_data = {'': ['*.txt', '*.rs', '*.pyx']},
     include_package_data = True,
     zip_safe = False,
-    python_requires = ">=3.6",
+    python_requires = ">=3.7",
     setup_requires = ['setuptools>=18.0', 'setuptools-rust'],
     install_requires = ["numpy"],
     extras_require = {

src/lib.rs

@@ -312,12 +312,12 @@ fn _decode_segment_into_frame(
             completely decoded"
         ).into()
     );
-    let err_eof = Err(
-        String::from(
-            "The end of the frame was reached before the segment was \
-            completely decoded"
-        ).into()
-    );
+    // let err_eof = Err(
+    //     String::from(
+    //         "The end of the frame was reached before the segment was \
+    //         completely decoded"
+    //     ).into()
+    // );
 
     loop {
         // `header_byte` is equivalent to N in the DICOM Standard
@@ -329,12 +329,21 @@ fn _decode_segment_into_frame(
             // however since using uint8 instead of int8 this will be
             // (256 - N + 1) times
             op_len = 257 - header_byte;
-            // Check we have enough encoded data and remaining frame
-            if (pos > max_offset) || (idx + op_len) > max_frame {
-                match pos > max_offset {
-                    true => return err_eod,
-                    false => return err_eof
+
+            // Check we have enough encoded data
+            if pos > max_offset {
+                return err_eod
+            }
+
+            // Check segment for excess padding
+            if (idx + op_len) > max_frame {
+                // Only copy until we reach the end of frame
+                for _ in 0..(max_frame - idx) {
+                    dst[idx] = src[pos];
+                    idx += bpp;
                 }
+
+                return Ok((idx - initial_offset) / bpp)
             }
 
             for _ in 0..op_len {
@@ -345,12 +354,21 @@ fn _decode_segment_into_frame(
         } else if header_byte < 128 {
             // Extend by literally copying the next (N + 1) bytes
             op_len = header_byte + 1;
-            // Check we have enough encoded data and remaining frame
-            if ((pos + header_byte) > max_offset) || (idx + op_len > max_frame) {
-                match (pos + header_byte) > max_offset {
-                    true => return err_eod,
-                    false => return err_eof
+
+            // Check we have enough encoded data
+            if (pos + header_byte) > max_offset {
+                return err_eod
+            }
+
+            // Check segment for excess padding
+            if (idx + op_len) > max_frame {
+                // Only extend until the end of frame
+                for ii in pos..pos + (max_frame - idx) {
+                    dst[idx] = src[ii];
+                    idx += bpp;
                 }
+
+                return Ok((idx - initial_offset) / bpp)
             }
 
             for ii in pos..pos + op_len {