Merge branch 'main' into feat/contrib-msgspec-cleanup

Author: cofin

docs/usage/middleware/builtin-middleware.rst

@@ -232,15 +232,15 @@ You can configure the following additional brotli-specific values:
 Zstd
 ^^^^^^
 
-The `zstandard <https://pypi.org/project/zstandard>`_ package is required to run this middleware. It is available as an extra for Litestar via the ``zstd`` extra: (``pip install 'litestar[zstd]'``).
+The `backports.zstd <https://pypi.org/project/backports.zstd/>`_ package is required to run this middleware. It is available as an extra for Litestar via the ``zstd`` extra: (``pip install 'litestar[zstd]'``).
 
 You can enable zstd compression of responses by passing an instance of
 :class:`~litestar.config.compression.CompressionConfig` with the ``backend`` parameter set to ``"zstd"``.
 
 You can configure the following additional zstd-specific values:
 
 * ``minimum_size``: the minimum threshold for response size to enable compression. Smaller responses will not be compressed. Default is 500, i.e. half a kilobyte.
-* ``zstd_compress_level``: Range [0-22], Controls the compression level. Higher values increase compression ratio but are slower. Default is 3.
+* ``zstd_compress_level``: Integer >= 0. Controls the compression level. Higher values increase compression ratio but are slower. A value of 0 indicates use of the library default, which is typically level 3. Default is 0.
 * ``zstd_gzip_fallback``: Boolean indicating whether to fall back to gzip if Zstd is not supported. Default is True.
 
 .. code-block:: python

docs/usage/requests.rst

@@ -304,7 +304,7 @@ To disable this limit for a specific handler / router / controller, it can be se
     For requests that define a ``Content-Length`` header, Litestar will not attempt to
     read the request body should the header value exceed the ``request_max_body_size``.
 
-    If the header value is within the allowed bounds, Litestar will verify during the
-    streaming of the request body that it does not exceed the size specified in the
-    header. Should the request exceed this size, it will abort the request with a
-    ``400 - Bad Request``.
+    Validation that the actual request body matches the ``Content-Length`` header is
+    delegated to the ASGI server (e.g. uvicorn). This ensures compatibility with
+    request decompression middleware, where the decompressed body may legitimately
+    exceed the wire-format ``Content-Length``.

litestar/config/compression.py

@@ -30,8 +30,10 @@ class CompressionConfig:
     """Minimum response size (bytes) to enable compression, affects all backends."""
     gzip_compress_level: int = field(default=9)
     """Range ``[0-9]``, see :doc:`python:library/gzip`."""
-    zstd_compress_level: int = field(default=3)
-    """Range `[0-22]`, see `zstandard <https://pypi.org/project/zstandard/>`"""
+    zstd_compress_level: int = field(default=0)
+    """Integer greater than or equal to 0.
+    A value of 0 indicates use of default compression level set by the library.
+    """
     brotli_quality: int = field(default=5)
     """Range ``[0-11]``, Controls the compression-speed vs compression-density tradeoff.
 
@@ -86,12 +88,13 @@ def __post_init__(self) -> None:
             self.gzip_fallback = self.brotli_gzip_fallback
             self.compression_facade = BrotliCompression
         elif self.backend == "zstd":
-            if self.zstd_compress_level < 1 or self.zstd_compress_level > 22:
+            from litestar.middleware.compression.zstd_facade import ZstdCompression
+
+            upper_bound = ZstdCompression.upper_bound
+            if not (0 <= self.zstd_compress_level <= upper_bound):
                 raise ImproperlyConfiguredException(
-                    f"zstd_compress_level must be between 1 and 22, given: {self.zstd_compress_level}"
+                    f"zstd_compress_level must be between 0 and {upper_bound}, given: {self.zstd_compress_level}"
                 )
 
-            from litestar.middleware.compression.zstd_facade import ZstdCompression
-
             self.gzip_fallback = self.zstd_gzip_fallback
             self.compression_facade = ZstdCompression

litestar/connection/request.py

@@ -208,23 +208,12 @@ async def stream(self) -> AsyncGenerator[bytes, None]:
                     if body:
                         total_bytes_streamed += len(body)
 
-                        # if a 'content-length' header was set, check if we have
-                        # received more bytes than specified. in most cases this should
-                        # be caught before it hits the application layer and an ASGI
-                        # server (e.g. uvicorn) will not allow this, but since it's not
-                        # forbidden according to the HTTP or ASGI spec, we err on the
-                        # side of caution and still perform this check.
-                        #
-                        # uvicorn documented behaviour for this case:
-                        # https://github.com/encode/uvicorn/blob/fe3910083e3990695bc19c2ef671dd447262ae18/docs/server-behavior.md?plain=1#L11
-                        if announced_content_length:
-                            if total_bytes_streamed > announced_content_length:
-                                raise ClientException("Malformed request")
-
-                        # we don't have a 'content-length' header, likely a chunked
-                        # transfer. we don't really care and simply check if we have
-                        # received more bytes than allowed
-                        elif total_bytes_streamed > max_content_length:
+                        # enforce the request body size limit on actual bytes
+                        # received. content-length validation is intentionally left to
+                        # the ASGI server (e.g. uvicorn) since middleware such as
+                        # request decompression can legitimately cause the streamed
+                        # body to exceed the wire-format content-length.
+                        if total_bytes_streamed > max_content_length:
                             raise RequestEntityTooLarge
 
                         yield body

litestar/middleware/compression/zstd_facade.py

@@ -1,15 +1,19 @@
 from __future__ import annotations
 
+import sys
 from typing import TYPE_CHECKING, Literal
 
 from litestar.enums import CompressionEncoding
 from litestar.exceptions import MissingDependencyException
 from litestar.middleware.compression.facade import CompressionFacade
 
-try:
-    import zstandard as zstd
-except ImportError as e:
-    raise MissingDependencyException("zstandard", extra="zstd") from e
+if sys.version_info >= (3, 14):
+    from compression import zstd
+else:
+    try:
+        from backports import zstd
+    except ImportError as e:
+        raise MissingDependencyException("backports.zstd", extra="zstd") from e
 
 if TYPE_CHECKING:
     from io import BytesIO
@@ -18,22 +22,31 @@
 
 
 class ZstdCompression(CompressionFacade):
-    __slots__ = ("buffer", "cctx", "compression_encoding", "compressor")
+    __slots__ = ("buffer", "compression_encoding", "compressor")
 
     encoding = CompressionEncoding("zstd")
-
-    def __init__(self, buffer: BytesIO, compression_encoding: Literal["zstd"] | str, config: CompressionConfig) -> None:
+    upper_bound = zstd.CompressionParameter.compression_level.bounds()[1]
+
+    def __init__(
+        self,
+        buffer: BytesIO,
+        compression_encoding: Literal["zstd"] | str,
+        config: CompressionConfig,
+    ) -> None:
         self.buffer = buffer
         self.compression_encoding = compression_encoding
-        self.cctx = zstd.ZstdCompressor(level=config.zstd_compress_level)
-        self.compressor = self.cctx.stream_writer(buffer)
-
-    def write(self, body: bytes | bytearray, final: bool = False) -> None:
-        self.compressor.write(body)
-        if final:
-            self.compressor.flush(zstd.FLUSH_FRAME)
+        self.compressor = zstd.ZstdCompressor(level=config.zstd_compress_level)
+
+    def write(
+        self,
+        body: bytes | bytearray,
+        final: bool = False,
+    ) -> None:
+        if not final:
+            self.buffer.write(self.compressor.compress(body, mode=zstd.ZstdCompressor.FLUSH_BLOCK))
         else:
-            self.compressor.flush(zstd.FLUSH_BLOCK)
+            self.buffer.write(self.compressor.compress(body, mode=zstd.ZstdCompressor.FLUSH_FRAME))
 
     def close(self) -> None:
-        self.compressor.flush(zstd.FLUSH_FRAME)
+        if self.compressor.last_mode != zstd.ZstdCompressor.FLUSH_FRAME:
+            self.buffer.write(self.compressor.flush())

pyproject.toml

@@ -53,7 +53,7 @@ annotated-types = ["annotated-types"]
 attrs = ["attrs"]
 brotli = ["brotli"]
 zstd = [
-    "zstandard>=0.24.0",
+    "backports-zstd>=1.3.0; python_version < '3.14'",
 ]
 cli = ["jsbeautifier", "uvicorn[standard]"]
 cryptography = ["cryptography"]
@@ -193,6 +193,7 @@ test = [
   "pytest-timeout",
   "pytest-xdist",
   "time-machine",
+  "zstandard",
 ]
 
 [build-system]

tests/unit/test_connection/test_request.py

@@ -538,15 +538,34 @@ async def get_state(request: Request[Any, Any, State]) -> dict[str, str]:
         assert response.json() == {"state": 2}
 
 
-def test_request_body_exceeds_content_length() -> None:
-    @post("/")
-    def handler(body: bytes) -> None:
-        pass
+def test_request_body_exceeding_content_length_is_allowed() -> None:
+    """Body larger than Content-Length but within request_max_body_size succeeds.
+
+    Decompression middleware can legitimately produce a body larger than the
+    wire-format Content-Length. Content-length enforcement is delegated to the
+    ASGI server (e.g. uvicorn).
+    """
+
+    @post("/", request_max_body_size=10)
+    def handler(body: bytes) -> bytes:
+        return body
 
     with create_test_client([handler]) as client:
         response = client.post("/", headers={"content-length": "1"}, content=b"ab")
-        assert response.status_code == HTTP_400_BAD_REQUEST
-        assert response.json() == {"status_code": 400, "detail": "Malformed request"}
+        assert response.status_code == 201
+        assert response.content == b"ab"
+
+
+def test_request_body_exceeding_content_length_still_enforces_max_body_size() -> None:
+    """Content-length mismatch is allowed, but request_max_body_size is still enforced."""
+
+    @post("/", request_max_body_size=1)
+    def handler(body: bytes) -> bytes:
+        return body
+
+    with create_test_client([handler]) as client:
+        response = client.post("/", headers={"content-length": "1"}, content=b"ab")
+        assert response.status_code == HTTP_413_REQUEST_ENTITY_TOO_LARGE
 
 
 def test_request_body_exceeds_max_request_body_size() -> None:

tests/unit/test_middleware/test_compression_middleware.py

@@ -1,5 +1,6 @@
 # pyright: reportUnnecessaryTypeIgnoreComment=false
 
+import sys
 import zlib
 from collections.abc import AsyncIterator, Callable
 from io import BytesIO
@@ -20,6 +21,12 @@
 from litestar.testing import create_test_client
 from litestar.types.asgi_types import ASGIApp, HTTPResponseBodyEvent, HTTPResponseStartEvent, Message, Scope
 
+if sys.version_info >= (3, 14):
+    from compression import zstd
+else:
+    from backports import zstd
+zstd_compression_level_upper_bound = zstd.CompressionParameter.compression_level.bounds()[1]
+
 BrotliMode = Literal["text", "generic", "font"]
 
 
@@ -165,7 +172,14 @@ def test_config_gzip_compress_level_validation(gzip_compress_level: int, should_
 
 
 @pytest.mark.parametrize(
-    "zstd_compress_level, should_raise", ((0, True), (1, False), (22, False), (23, True), (-1, True))
+    "zstd_compress_level, should_raise",
+    (
+        (-1, True),
+        (0, False),
+        (1, False),
+        (zstd_compression_level_upper_bound, False),
+        (zstd_compression_level_upper_bound + 1, True),
+    ),
 )
 def test_config_zstd_compress_level_validation(zstd_compress_level: int, should_raise: bool) -> None:
     if should_raise: