[ChunkCodecCore] BREAKING change to the resizing behavior in try_resize_decode! (#45)

* change resize logic

* update changelog

* Add unit tests for `grow_dst!`

Author: nhz2

ChunkCodecCore/CHANGELOG.md

@@ -6,6 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+### BREAKING the resizing behavior in `try_resize_decode!` changed [#45](https://github.com/JuliaIO/ChunkCodecs.jl/pull/45)
+
+`dst` may now be longer than the returned number of bytes, even if `dst` was grown with `resize!`.
+
+`try_resize_decode!` will now only grow `dst`, never shrink it.
+
+If `max_size` is less than `length(dst)`, instead of throwing an error, `try_resize_decode!` will now act as if `max_size == length(dst)`.
+
+- Added the `grow_dst!` helper function. [#45](https://github.com/JuliaIO/ChunkCodecs.jl/pull/45)
+
 ## [v0.4.2](https://github.com/JuliaIO/ChunkCodecs.jl/tree/ChunkCodecCore-v0.4.2) - 2025-04-07
 
 - Added the `decode!` function. [#41](https://github.com/JuliaIO/ChunkCodecs.jl/pull/41)

ChunkCodecCore/Project.toml

@@ -1,7 +1,7 @@
 name = "ChunkCodecCore"
 uuid = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1"
 authors = ["nhz2 <nhz2@cornell.edu>"]
-version = "0.4.2"
+version = "0.5.0-dev"
 
 [compat]
 julia = "1.10"

ChunkCodecCore/src/ChunkCodecCore.jl

@@ -24,6 +24,7 @@ if VERSION >= v"1.11.0-DEV.469"
 
             check_in_range,
             check_contiguous,
+            grow_dst!,
 
             can_concatenate,
             is_thread_safe,

ChunkCodecCore/src/interface.jl

@@ -60,10 +60,8 @@ function decode(
         throw(DecodedSizeError(_clamp_max_size, try_find_decoded_size(d, src)))
     end
     @assert real_dst_size ∈ 0:_clamp_max_size
-    if real_dst_size < _clamp_size_hint
-        resize!(dst, real_dst_size)
-    end
-    @assert real_dst_size == length(dst)
+    @assert real_dst_size ≤ length(dst)
+    resize!(dst, real_dst_size)
     dst
 end
 
@@ -206,55 +204,32 @@ Try to decode the input `src` into `dst` using decoder `d`.
 
 Return the size of the decoded output in `dst` if successful.
 
-If `dst` is too small, resize `dst` and try again.
-If the `max_size` limit will be passed, return `nothing`.
-
-`dst` can be resized using the `resize!` function to any size between the original length of `dst` and `max_size`.
+`dst` can be grown using the `resize!` function to any size between one more than the original length of `dst` and `max_size`.
 
-Throw an `ArgumentError` if `length(dst)` is not in `0:max_size`.
+Return `nothing` if the size of `dst` is too small to contain the decoded output and cannot be grown due to the `max_size` restriction.
 
 Precondition: `dst` and `src` do not overlap in memory.
 
 All of `dst` can be written to or used as scratch space by the decoder.
 Only the initial returned number of bytes are valid output.
-
-If the size of `dst` is increased, its length will be equal to the returned number of bytes.
 """
 function try_resize_decode!(d, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}, max_size::Int64; kwargs...)::Union{Nothing, Int64}
-    check_in_range(Int64(0):max_size; dst_size=length(dst))
-    olb::Int64 = length(dst)
     decoded_size = try_find_decoded_size(d, src)::Union{Nothing, Int64}
     if isnothing(decoded_size)
         while true
             ds = try_decode!(d, dst, src)::Union{Nothing, Int64}
             if isnothing(ds)
-                # grow dst
-                local cur_size::Int64 = length(dst)
-                if cur_size == max_size
-                    return
-                end
-                @assert cur_size < max_size
-                # This inequality prevents overflow
-                local next_size = if max_size - cur_size ≤ cur_size
-                    max_size
-                else
-                    max(2*cur_size, Int64(1))
-                end
-                resize!(dst, next_size)
+                @something grow_dst!(dst, max_size) return nothing
             else
                 @assert ds ∈ 0:length(dst)
-                if length(dst) > olb && length(dst) != ds
-                    @assert ds > olb
-                    resize!(dst, ds) # shrink to just contain output if it was resized.
-                end
                 return ds
             end
         end
     else
-        if decoded_size ∉ 0:max_size
-            return
-        end
-        if decoded_size > olb
+        if decoded_size > length(dst)
+            if decoded_size > max_size
+                return nothing
+            end
             resize!(dst, decoded_size)
         end
         real_dst_size = something(try_decode!(d, dst, src))
@@ -300,3 +275,24 @@ function check_in_range(range; kwargs...)
         end
     end
 end
+
+"""
+    grow_dst!(dst::AbstractVector{UInt8}, max_size::Int64)::Union{Nothing, Int64}
+
+Grow the destination vector `dst` to a size between its current size and `max_size`.
+Return the new size of `dst` if it was grown, or `nothing` if it could not be grown due to the `max_size` restriction.
+"""
+function grow_dst!(dst::AbstractVector{UInt8}, max_size::Int64)::Union{Nothing, Int64}
+    cur_size::Int64 = length(dst)
+    if cur_size ≥ max_size
+        return
+    end
+    # This inequality prevents overflow
+    next_size::Int64 = if max_size - cur_size ≤ cur_size
+        max_size
+    else
+        max(2*cur_size, Int64(1))
+    end
+    resize!(dst, next_size)
+    return next_size
+end

ChunkCodecCore/test/runtests.jl

@@ -53,6 +53,21 @@ end
     @test_throws ArgumentError ChunkCodecCore.check_in_range(1:6; x=7)
     @test isnothing(ChunkCodecCore.check_in_range(1:6; x=6))
     @test isnothing(ChunkCodecCore.check_in_range(1:6; x=1))
+
+    x = zeros(UInt8, 0)
+    for m in [typemin(Int64), Int64(-1), Int64(0)]
+        @test isnothing(ChunkCodecCore.grow_dst!(x, m))
+        @test length(x) == 0
+    end
+    @test ChunkCodecCore.grow_dst!(x, Int64(1)) === Int64(1)
+    @test length(x) == 1
+    @test ChunkCodecCore.grow_dst!(x, typemax(Int64)) == length(x)
+    n1 = length(x)
+    @test n1 > 1
+    @test isnothing(ChunkCodecCore.grow_dst!(x, Int64(n1)))
+    @test length(x) == n1
+    @test ChunkCodecCore.grow_dst!(x, Int64(n1 + 1)) == n1 + 1
+    @test length(x) == n1 + 1
 end
 
 # version of NoopDecodeOptions that returns unknown try_find_decoded_size

ChunkCodecTests/Project.toml

@@ -8,6 +8,6 @@ ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
-ChunkCodecCore = "0.4"
+ChunkCodecCore = "0.5"
 Test = "1"
 julia = "1.10"

ChunkCodecTests/src/ChunkCodecTests.jl

@@ -110,10 +110,8 @@ function test_encoder_decoder(e, d; trials=100)
 
         if s > 0
             dst = zeros(UInt8, s - 1)
-            @test_throws(
-                ArgumentError("dst_size ∈ $(0:-1) must hold. Got\ndst_size => $(s-1)"),
-                try_resize_decode!(d, dst, encoded, Int64(-1))
-            )
+            @test isnothing(try_resize_decode!(d, dst, encoded, Int64(-1)))
+            @test length(dst) == s - 1
             dst = zeros(UInt8, s - 1)
             @test try_resize_decode!(d, dst, encoded, s) == s
             @test length(dst) == s
@@ -130,10 +128,8 @@ function test_encoder_decoder(e, d; trials=100)
         end
         dst_buffer = zeros(UInt8, s + 2)
         dst = view(dst_buffer, 1:s+1)
-        @test_throws(
-            ArgumentError("dst_size ∈ $(0:s) must hold. Got\ndst_size => $(s+1)"),
-            try_resize_decode!(d, dst, encoded, s),
-        )
+        @test try_resize_decode!(d, dst, encoded, s-1) === s
+        @test try_resize_decode!(d, dst, encoded, s) === s
         @test try_resize_decode!(d, dst, encoded, s+2) === s
         @test length(dst) == s + 1
         @test dst[1:s] == data

LibBlosc/Project.toml

@@ -9,7 +9,7 @@ ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1"
 
 [compat]
 Blosc_jll = "1"
-ChunkCodecCore = "0.4"
+ChunkCodecCore = "0.5"
 julia = "1.10"
 
 [workspace]

LibBrotli/Project.toml

@@ -8,7 +8,7 @@ ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1"
 brotli_jll = "4611771a-a7d2-5e23-8d00-b1becdba1aae"
 
 [compat]
-ChunkCodecCore = "0.4"
+ChunkCodecCore = "0.5"
 brotli_jll = "1"
 julia = "1.10"
 

LibBrotli/src/ChunkCodecLibBrotli.jl

@@ -8,6 +8,7 @@ using ChunkCodecCore:
     DecodeOptions,
     check_contiguous,
     check_in_range,
+    grow_dst!,
     DecodingError
 import ChunkCodecCore:
     decode_options,