Skip to content

Commit 3ffc40f

Browse files
nhz2nhz2
authored
[LibBrotli] Add brotli format (#43)
1 parent a255960 commit 3ffc40f

17 files changed

+725
-1
lines changed

‎.github/workflows/CI.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,11 @@ jobs:
3737
- ChunkCodecCore/**
3838
- ChunkCodecTests/**
3939
- LibBlosc/**
40+
LibBrotli:
41+
- .github/**
42+
- ChunkCodecCore/**
43+
- ChunkCodecTests/**
44+
- LibBrotli/**
4045
LibBzip2:
4146
- .github/**
4247
- ChunkCodecCore/**

‎CONTRIBUTING.md

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,3 +59,23 @@ For example to run
5959
julia --project=test -e 'import Pkg; Pkg.update()'
6060
julia --project=test test/imagecodecs-compat.jl
6161
```
62+
63+
## Creating a new ChunkCodec package
64+
65+
Start by generating a new subdirectory package.
66+
67+
If the new package is wrapping a C library use the `Lib` prefix.
68+
69+
For example:
70+
71+
```julia-repl
72+
julia> using Pkg; Pkg.generate("LibFoo")
73+
```
74+
75+
Add this subdirectory to the "workspace" section of the root "Project.toml"
76+
77+
Add the package to the ".github/workflows/CI.yml" file
78+
79+
Adjust the new subdirectory to match the style of the existing subdirectories.
80+
"LibBzip2" is a good example of a streaming format. "LibSnappy" is a good example
81+
of a non streaming format.

‎LibBrotli/CHANGELOG.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Release Notes
2+
3+
All notable changes to this package will be documented in this file.
4+
5+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
6+
7+
## Unreleased
8+
9+
### Added
10+
11+
- Initial release

‎LibBrotli/LICENSE

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2025 Nathan Zimmerberg
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎LibBrotli/Project.toml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name = "ChunkCodecLibBrotli"
2+
uuid = "653b0ff7-85b5-4442-93c1-dcc330d3ec7d"
3+
authors = ["nhz2 <[email protected]>"]
4+
version = "0.1.0"
5+
6+
[deps]
7+
ChunkCodecCore = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1"
8+
brotli_jll = "4611771a-a7d2-5e23-8d00-b1becdba1aae"
9+
10+
[compat]
11+
ChunkCodecCore = "0.4"
12+
brotli_jll = "1"
13+
julia = "1.10"
14+
15+
[workspace]
16+
projects = ["test"]

‎LibBrotli/README.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# ChunkCodecLibBrotli
2+
3+
## Warning: ChunkCodecLibBrotli is currently a WIP and its API may drastically change at any time.
4+
5+
This package implements the ChunkCodec interface for the following encoders and decoders
6+
using the brotli C library <https://brotli.org/>
7+
8+
1. `BrotliCodec`, `BrotliEncodeOptions`, `BrotliDecodeOptions`
9+
10+
## Example
11+
12+
```julia-repl
13+
julia> using ChunkCodecLibBrotli
14+
15+
julia> data = [0x00, 0x01, 0x02, 0x03];
16+
17+
julia> compressed_data = encode(BrotliEncodeOptions(;quality=6), data);
18+
19+
julia> decompressed_data = decode(BrotliCodec(), compressed_data; max_size=length(data), size_hint=length(data));
20+
21+
julia> data == decompressed_data
22+
true
23+
```
24+
25+
The low level interface is defined in the `ChunkCodecCore` package.
26+

‎LibBrotli/src/ChunkCodecLibBrotli.jl

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
module ChunkCodecLibBrotli
2+
3+
using brotli_jll: libbrotlidec, libbrotlienc
4+
5+
using ChunkCodecCore:
6+
Codec,
7+
EncodeOptions,
8+
DecodeOptions,
9+
check_contiguous,
10+
check_in_range,
11+
DecodingError
12+
import ChunkCodecCore:
13+
decode_options,
14+
try_decode!,
15+
try_resize_decode!,
16+
try_encode!,
17+
encode_bound,
18+
is_thread_safe,
19+
try_find_decoded_size,
20+
decoded_size_range
21+
22+
export BrotliCodec,
23+
BrotliEncodeOptions,
24+
BrotliDecodeOptions,
25+
BrotliDecodingError
26+
27+
# reexport ChunkCodecCore
28+
using ChunkCodecCore: ChunkCodecCore, encode, decode
29+
export ChunkCodecCore, encode, decode
30+
31+
32+
include("libbrotli.jl")
33+
34+
"""
35+
struct BrotliCodec <: Codec
36+
BrotliCodec()
37+
38+
brotli compression using the brotli C library <https://brotli.org/>
39+
40+
This is the brotli (.br) format described in RFC 7932
41+
42+
See also [`BrotliEncodeOptions`](@ref) and [`BrotliDecodeOptions`](@ref)
43+
"""
44+
struct BrotliCodec <: Codec
45+
end
46+
decode_options(::BrotliCodec) = BrotliDecodeOptions()
47+
48+
include("encode.jl")
49+
include("decode.jl")
50+
51+
end # module ChunkCodecLibBrotli

‎LibBrotli/src/decode.jl

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
"""
2+
BrotliDecodingError(msg)
3+
4+
Error for data that cannot be decoded.
5+
"""
6+
struct BrotliDecodingError <: DecodingError
7+
msg::String
8+
end
9+
10+
function Base.showerror(io::IO, err::BrotliDecodingError)
11+
print(io, "BrotliDecodingError: ")
12+
print(io, err.msg)
13+
nothing
14+
end
15+
16+
"""
17+
struct BrotliDecodeOptions <: DecodeOptions
18+
BrotliDecodeOptions(; kwargs...)
19+
20+
brotli decompression using the brotli C library <https://brotli.org/>
21+
22+
This is the brotli (.br) format described in RFC 7932
23+
24+
# Keyword Arguments
25+
26+
- `codec::BrotliCodec=BrotliCodec()`
27+
"""
28+
struct BrotliDecodeOptions <: DecodeOptions
29+
codec::BrotliCodec
30+
end
31+
function BrotliDecodeOptions(;
32+
codec::BrotliCodec=BrotliCodec(),
33+
kwargs...
34+
)
35+
BrotliDecodeOptions(codec)
36+
end
37+
38+
# https://github.com/google/brotli/issues/501
39+
is_thread_safe(::BrotliDecodeOptions) = true
40+
41+
function try_find_decoded_size(::BrotliDecodeOptions, src::AbstractVector{UInt8})::Nothing
42+
nothing
43+
end
44+
45+
function try_decode!(d::BrotliDecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}; kwargs...)::Union{Nothing, Int64}
46+
try_resize_decode!(d, dst, src, Int64(length(dst)))
47+
end
48+
49+
function try_resize_decode!(d::BrotliDecodeOptions, dst::AbstractVector{UInt8}, src::AbstractVector{UInt8}, max_size::Int64; kwargs...)::Union{Nothing, Int64}
50+
check_in_range(Int64(0):max_size; dst_size=length(dst))
51+
olb::Int64 = length(dst)
52+
dst_size::Int64 = olb
53+
src_size::Int64 = length(src)
54+
src_left::Int64 = src_size
55+
dst_left::Int64 = dst_size
56+
check_contiguous(dst)
57+
check_contiguous(src)
58+
if isempty(src)
59+
throw(BrotliDecodingError("unexpected end of stream"))
60+
end
61+
s = @ccall libbrotlidec.BrotliDecoderCreateInstance(
62+
C_NULL::Ptr{Cvoid},
63+
C_NULL::Ptr{Cvoid},
64+
C_NULL::Ptr{Cvoid},
65+
)::Ptr{BrotliDecoderState}
66+
if s == C_NULL
67+
throw(OutOfMemoryError())
68+
end
69+
try
70+
cconv_src = Base.cconvert(Ptr{UInt8}, src)
71+
while true
72+
# dst may get resized, so cconvert needs to be redone on each iteration.
73+
cconv_dst = Base.cconvert(Ptr{UInt8}, dst)
74+
GC.@preserve cconv_src cconv_dst begin
75+
src_p = Base.unsafe_convert(Ptr{UInt8}, cconv_src)
76+
dst_p = Base.unsafe_convert(Ptr{UInt8}, cconv_dst)
77+
available_in = Ref(Csize_t(src_left))
78+
next_in = src_p + (src_size - src_left)
79+
available_out = Ref(Csize_t(dst_left))
80+
next_out = dst_p + (dst_size - dst_left)
81+
result = @ccall libbrotlidec.BrotliDecoderDecompressStream(
82+
s::Ptr{BrotliDecoderState}, # state
83+
available_in::Ref{Csize_t}, # available_in
84+
next_in::Ref{Ptr{UInt8}}, # next_in
85+
available_out::Ref{Csize_t}, # available_out
86+
next_out::Ref{Ptr{UInt8}}, # next_out
87+
C_NULL::Ptr{Csize_t}, # total_out
88+
)::Cint
89+
if result == BROTLI_DECODER_RESULT_SUCCESS || result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT
90+
@assert available_in[] ≤ src_left
91+
@assert available_out[] ≤ dst_left
92+
src_left = available_in[]
93+
dst_left = available_out[]
94+
@assert src_left ∈ 0:src_size
95+
@assert dst_left ∈ 0:dst_size
96+
end
97+
if result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT
98+
@assert iszero(dst_left)
99+
# grow dst or return nothing
100+
if dst_size ≥ max_size
101+
return nothing
102+
end
103+
# This inequality prevents overflow
104+
local next_size = if max_size - dst_size ≤ dst_size
105+
max_size
106+
else
107+
max(2*dst_size, Int64(1))
108+
end
109+
resize!(dst, next_size)
110+
dst_left += next_size - dst_size
111+
dst_size = next_size
112+
@assert dst_left > 0
113+
elseif result == BROTLI_DECODER_RESULT_SUCCESS
114+
if iszero(src_left)
115+
# yay done return decompressed size
116+
real_dst_size = dst_size - dst_left
117+
@assert real_dst_size ∈ 0:length(dst)
118+
if length(dst) > olb # shrink to just contain output if it was resized.
119+
resize!(dst, real_dst_size)
120+
end
121+
return real_dst_size
122+
else
123+
# Otherwise, throw an error
124+
throw(BrotliDecodingError("unexpected $(src_left) bytes after stream"))
125+
end
126+
elseif result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT
127+
throw(BrotliDecodingError("unexpected end of stream"))
128+
elseif result == BROTLI_DECODER_RESULT_ERROR
129+
err_code = @ccall libbrotlidec.BrotliDecoderGetErrorCode(
130+
s::Ptr{BrotliDecoderState}, # state
131+
)::Cint
132+
if err_code ∈ RANGE_BROTLI_DECODER_ERROR_ALLOC
133+
throw(OutOfMemoryError())
134+
else
135+
err_str = @ccall libbrotlidec.BrotliDecoderErrorString(
136+
err_code::Cint,
137+
)::Ptr{Cchar}
138+
throw(BrotliDecodingError(unsafe_string(err_str)))
139+
end
140+
else
141+
error("unknown brotli decoder result: $(result)")
142+
end
143+
end
144+
end
145+
finally
146+
@ccall libbrotlidec.BrotliDecoderDestroyInstance(
147+
s::Ptr{BrotliDecoderState},
148+
)::Cvoid
149+
end
150+
end

0 commit comments

Comments
 (0)