[LibBrotli] Add brotli format

.github/workflows/CI.yml

@@ -37,6 +37,11 @@ jobs:
             - ChunkCodecCore/**
             - ChunkCodecTests/**
             - LibBlosc/**
+          LibBrotli:
+            - .github/**
+            - ChunkCodecCore/**
+            - ChunkCodecTests/**
+            - LibBrotli/**
           LibBzip2:
             - .github/**
             - ChunkCodecCore/**

CONTRIBUTING.md

@@ -59,3 +59,23 @@ For example to run
 julia --project=test -e 'import Pkg; Pkg.update()'
 julia --project=test test/imagecodecs-compat.jl
 ```
+
+## Creating a new ChunkCodec package
+
+Start by generating a new subdirectory package.
+
+If the new package is wrapping a C library use the `Lib` prefix.
+
+For example:
+
+```julia-repl
+julia> using Pkg; Pkg.generate("LibFoo")
+```
+
+Add this subdirectory to the "workspace" section of the root "Project.toml"
+
+Add the package to the ".github/workflows/CI.yml" file
+
+Adjust the new subdirectory to match the style of the existing subdirectories.
+"LibBzip2" is a good example of a streaming format. "LibSnappy" is a good example
+of a non streaming format.

LibBrotli/CHANGELOG.md

@@ -0,0 +1,11 @@
+# Release Notes
+
+All notable changes to this package will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+## Unreleased
+
+### Added
+
+- Initial release

LibBrotli/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Nathan Zimmerberg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LibBrotli/Project.toml

@@ -0,0 +1,16 @@
+name = "ChunkCodecLibBrotli"
+uuid = "653b0ff7-85b5-4442-93c1-dcc330d3ec7d"
+authors = ["nhz2 <[email protected]>"]
+version = "0.1.0"
+
+[deps]
+ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1"
+brotli_jll = "4611771a-a7d2-5e23-8d00-b1becdba1aae"
+
+[compat]
+ChunkCodecCore = "0.4"
+brotli_jll = "1"
+julia = "1.10"
+
+[workspace]
+projects = ["test"]

LibBrotli/README.md

@@ -0,0 +1,26 @@
+# ChunkCodecLibBrotli
+
+## Warning: ChunkCodecLibBrotli is currently a WIP and its API may drastically change at any time.
+
+This package implements the ChunkCodec interface for the following encoders and decoders
+using the brotli C library <https://brotli.org/>
+
+1. `BrotliCodec`, `BrotliEncodeOptions`, `BrotliDecodeOptions`
+
+## Example
+
+```julia-repl
+julia> using ChunkCodecLibBrotli
+
+julia> data = [0x00, 0x01, 0x02, 0x03];
+
+julia> compressed_data = encode(BrotliEncodeOptions(;quality=6), data);
+
+julia> decompressed_data = decode(BrotliCodec(), compressed_data; max_size=length(data), size_hint=length(data));
+
+julia> data == decompressed_data
+true
+```
+
+The low level interface is defined in the `ChunkCodecCore` package.
+

LibBrotli/src/ChunkCodecLibBrotli.jl

@@ -0,0 +1,51 @@
+module ChunkCodecLibBrotli
+
+using brotli_jll: libbrotlidec, libbrotlienc
+
+using ChunkCodecCore:
+    Codec,
+    EncodeOptions,
+    DecodeOptions,
+    check_contiguous,
+    check_in_range,
+    DecodingError
+import ChunkCodecCore:
+    decode_options,
+    try_decode!,
+    try_resize_decode!,
+    try_encode!,
+    encode_bound,
+    is_thread_safe,
+    try_find_decoded_size,
+    decoded_size_range
+
+export BrotliCodec,
+    BrotliEncodeOptions,
+    BrotliDecodeOptions,
+    BrotliDecodingError
+
+# reexport ChunkCodecCore
+using ChunkCodecCore: ChunkCodecCore, encode, decode
+export ChunkCodecCore, encode, decode
+
+
+include("libbrotli.jl")
+
+"""
+    struct BrotliCodec <: Codec
+    BrotliCodec()
+
+brotli compression using the brotli C library <https://brotli.org/>
+
+This is the brotli (.br) format described in RFC 7932
+
+See also [`BrotliEncodeOptions`](@ref) and [`BrotliDecodeOptions`](@ref)
+"""
+struct BrotliCodec <: Codec
+end
+decode_options(::BrotliCodec) = BrotliDecodeOptions()
+
+include("encode.jl")
+include("decode.jl")
+
+end # module ChunkCodecLibBrotli

LibBrotli/src/decode.jl

@@ -0,0 +1,150 @@
+"""
+    BrotliDecodingError(msg)
+
+Error for data that cannot be decoded.
+"""
+struct BrotliDecodingError <: DecodingError
+    msg::String
+end
+
+function Base.showerror(io::IO, err::BrotliDecodingError)
+    print(io, "BrotliDecodingError: ")
+    print(io, err.msg)
+    nothing
+end
+
+"""
+    struct BrotliDecodeOptions <: DecodeOptions
+    BrotliDecodeOptions(; kwargs...)
+
+brotli decompression using the brotli C library <https://brotli.org/>
+
+This is the brotli (.br) format described in RFC 7932
+
+# Keyword Arguments
+
+- `codec::BrotliCodec=BrotliCodec()`
+"""
+struct BrotliDecodeOptions <: DecodeOptions
+    codec::BrotliCodec
+end
+function BrotliDecodeOptions(;
+        codec::BrotliCodec=BrotliCodec(),
+        kwargs...
+    )
+    BrotliDecodeOptions(codec)
+end
+
+# https://github.com/google/brotli/issues/501
+is_thread_safe(::BrotliDecodeOptions) = true
+
+function try_find_decoded_size(::BrotliDecodeOptions, src::AbstractVector{UInt8})::Nothing
+    nothing
+end
+
+function try_decode!(d::BrotliDecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; kwargs...)::Union{Nothing, Int64}
+    try_resize_decode!(d, dst, src, Int64(length(dst)))
+end
+
+function try_resize_decode!(d::BrotliDecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}, max_size::Int64; kwargs...)::Union{Nothing, Int64}
+    check_in_range(Int64(0):max_size; dst_size=length(dst))
+    olb::Int64 = length(dst)
+    dst_size::Int64 = olb
+    src_size::Int64 = length(src)
+    src_left::Int64 = src_size
+    dst_left::Int64 = dst_size
+    check_contiguous(dst)
+    check_contiguous(src)
+    if isempty(src)
+        throw(BrotliDecodingError("unexpected end of stream"))
+    end
+    s = @ccall libbrotlidec.BrotliDecoderCreateInstance(
+        C_NULL::Ptr{Cvoid},
+        C_NULL::Ptr{Cvoid},
+        C_NULL::Ptr{Cvoid},
+    )::Ptr{BrotliDecoderState}
+    if s == C_NULL
+        throw(OutOfMemoryError())
+    end
+    try
+        cconv_src = Base.cconvert(Ptr{UInt8}, src)
+        while true
+            # dst may get resized, so cconvert needs to be redone on each iteration.
+            cconv_dst = Base.cconvert(Ptr{UInt8}, dst)
+            GC.@preserve cconv_src cconv_dst begin
+                src_p = Base.unsafe_convert(Ptr{UInt8}, cconv_src)
+                dst_p = Base.unsafe_convert(Ptr{UInt8}, cconv_dst)
+                available_in = Ref(Csize_t(src_left))
+                next_in = src_p + (src_size - src_left)
+                available_out = Ref(Csize_t(dst_left))
+                next_out = dst_p + (dst_size - dst_left)
+                result = @ccall libbrotlidec.BrotliDecoderDecompressStream(
+                    s::Ptr{BrotliDecoderState}, # state
+                    available_in::Ref{Csize_t}, # available_in
+                    next_in::Ref{Ptr{UInt8}}, # next_in
+                    available_out::Ref{Csize_t}, # available_out
+                    next_out::Ref{Ptr{UInt8}}, # next_out
+                    C_NULL::Ptr{Csize_t}, # total_out
+                )::Cint
+                if result == BROTLI_DECODER_RESULT_SUCCESS || result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT
+                    @assert available_in[] ≤ src_left
+                    @assert available_out[] ≤ dst_left
+                    src_left = available_in[]
+                    dst_left = available_out[]
+                    @assert src_left ∈ 0:src_size
+                    @assert dst_left ∈ 0:dst_size
+                end
+                if result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT
+                    @assert iszero(dst_left)
+                    # grow dst or return nothing
+                    if dst_size ≥ max_size
+                        return nothing
+                    end
+                    # This inequality prevents overflow
+                    local next_size = if max_size - dst_size ≤ dst_size
+                        max_size
+                    else
+                        max(2*dst_size, Int64(1))
+                    end
+                    resize!(dst, next_size)
+                    dst_left += next_size - dst_size
+                    dst_size = next_size
+                    @assert dst_left > 0
+                elseif result == BROTLI_DECODER_RESULT_SUCCESS
+                    if iszero(src_left)
+                        # yay done return decompressed size
+                        real_dst_size = dst_size - dst_left
+                        @assert real_dst_size ∈ 0:length(dst)
+                        if length(dst) > olb # shrink to just contain output if it was resized.
+                            resize!(dst, real_dst_size)
+                        end
+                        return real_dst_size
+                    else
+                        # Otherwise, throw an error
+                        throw(BrotliDecodingError("unexpected $(src_left) bytes after stream"))
+                    end
+                elseif result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT
+                    throw(BrotliDecodingError("unexpected end of stream"))
+                elseif result == BROTLI_DECODER_RESULT_ERROR
+                    err_code = @ccall libbrotlidec.BrotliDecoderGetErrorCode(
+                        s::Ptr{BrotliDecoderState}, # state
+                    )::Cint
+                    if err_code ∈ RANGE_BROTLI_DECODER_ERROR_ALLOC
+                        throw(OutOfMemoryError())
+                    else
+                        err_str = @ccall libbrotlidec.BrotliDecoderErrorString(
+                            err_code::Cint,
+                        )::Ptr{Cchar}
+                        throw(BrotliDecodingError(unsafe_string(err_str)))
+                    end
+                else
+                    error("unknown brotli decoder result: $(result)")
+                end
+            end
+        end
+    finally
+        @ccall libbrotlidec.BrotliDecoderDestroyInstance(
+            s::Ptr{BrotliDecoderState},
+        )::Cvoid
+    end
+end