Skip to content

Commit ba6cd53

Browse files
neueccneuecc
authored
Merge pull request #22 from Cysharp/compression
Brotli compression support
2 parents ec63bc4 + db797d8 commit ba6cd53

File tree

12 files changed

+959
-16
lines changed

12 files changed

+959
-16
lines changed

‎README.md

+39-1
Original file line numberDiff line numberDiff line change
@@ -487,7 +487,45 @@ float and double are 4 bytes and 8 bytes in MemoryPack, but 5 bytes and 9 bytes
487487

488488
String is UTF8 by default, which is similar to other serializers, but if the UTF16 option is chosen, it will be of a different nature.
489489

490-
In any case, if the payload size is large, compression should be considered. LZ4, ZStandard and Brotli are recommended. An efficient way to combine compression and serialization will be presented at a later date.
490+
In any case, if the payload size is large, compression should be considered. LZ4, ZStandard and Brotli are recommended.
491+
492+
### Compression
493+
494+
MemoryPack provides an efficient helper for [Brotli](https://github.com/google/brotli) compression via [BrotliEncoder](https://learn.microsoft.com/en-us/dotnet/api/system.io.compression.brotliencoder) and [BrotliDecoder](https://learn.microsoft.com/en-us/dotnet/api/system.io.compression.brotlidecoder). MemoryPack's `BrotliCompressor` and `BrotliDecompressor` provide compression/decompression optimized for MemoryPack's internal behavior.
495+
496+
```csharp
497+
using MemoryPack.Compression;
498+
499+
// Compression(require using)
500+
using var compressor = new BrotliCompressor();
501+
MemoryPackSerializer.Serialize(compressor, value);
502+
503+
// Get compressed byte[]
504+
var compressedBytes = compressor.ToArray();
505+
506+
// Or write to other IBufferWriter<byte>(for example PipeWriter)
507+
compressor.CopyTo(response.BodyWriter);
508+
```
509+
510+
```csharp
511+
using MemoryPack.Compression;
512+
513+
// Decompression(require using)
514+
using var decompressor = new BrotliDecompressor();
515+
516+
// Get decompressed ReadOnlySequence<byte> from ReadOnlySpan<byte> or ReadOnlySequence<byte>
517+
var decompressedBuffer = decompressor.Decompress(buffer);
518+
519+
var value = MemoryPackSerializer.Deserialize<T>(decompressedBuffer);
520+
```
521+
522+
Both `BrotliCompressor` and `BrotliDecompressor` are struct, it does not allocate memory on heap. Both store compressed or decompressed data in an internal memory pool for Serialize/Deserialize. Therefore, it is necessary to release the memory pooling, don't forget to use `using`.
523+
524+
Compression level is very important. The default is set to quality-1 (CompressionLevel.Fastest), which is different from the .NET default (CompressionLevel.Optimal, quality-4).
525+
526+
Fastest (quality-1) will be close to the speed of [LZ4](https://github.com/lz4/lz4), but 4 is much slower. This was determined to be critical in the serializer use scenario. Be careful when using the standard `BrotliStream`(quality-4 is the default). In any case, compression/decompression speeds and sizes will result in very different results for different data. Please prepare the data to be handled by your application and test it yourself.
527+
528+
Note that there is a several-fold speed penalty between MemoryPack's uncompressed and Brotli's added compression.
491529

492530
Packages
493531
---

‎sandbox/Benchmark/Benchmark.csproj

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
@@ -24,6 +24,8 @@
2424
<ItemGroup>
2525
<PackageReference Include="BenchmarkDotNet" Version="0.13.2" />
2626
<PackageReference Include="BinaryPack" Version="1.0.3" />
27+
<PackageReference Include="K4os.Compression.LZ4" Version="1.2.16" />
28+
<PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.2.16" />
2729
<PackageReference Include="MessagePack" Version="2.4.35" />
2830
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
2931
<PackageReference Include="Microsoft.Orleans.CodeGenerator" Version="4.0.0-preview2">

‎sandbox/Benchmark/BenchmarkNetUtilities/Categories.cs

+2
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,6 @@ public static class Categories
1010
{
1111
public const string Bytes = " byte[]";
1212
public const string BufferWriter = "BufferWriter";
13+
public const string Serialize = "Serialize";
14+
public const string Deserialize = "Deserialize";
1315
}

‎sandbox/Benchmark/Benchmarks/Compression.cs

+256
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
using Benchmark.BenchmarkNetUtilities;
2+
using BenchmarkDotNet.Configs;
3+
using K4os.Compression.LZ4.Encoders;
4+
using K4os.Compression.LZ4.Streams;
5+
using MemoryPack;
6+
using MemoryPack.Compression;
7+
using System.IO.Compression;
8+
9+
namespace Benchmark.Benchmarks;
10+
11+
[CategoriesColumn]
12+
[PayloadColumn]
13+
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
14+
public class Compression<T> : SerializerTestBase<T>
15+
{
16+
MemoryStream ms;
17+
LZ4EncoderSettings Fast;
18+
LZ4EncoderSettings L10Opt;
19+
LZ4EncoderSettings L04HC;
20+
byte[] normal;
21+
byte[] brotliFast;
22+
byte[] lz4Fast;
23+
24+
public Compression()
25+
: base()
26+
{
27+
ms = new MemoryStream();
28+
Fast = new LZ4EncoderSettings { CompressionLevel = K4os.Compression.LZ4.LZ4Level.L00_FAST };
29+
L10Opt = new LZ4EncoderSettings { CompressionLevel = K4os.Compression.LZ4.LZ4Level.L10_OPT };
30+
L04HC = new LZ4EncoderSettings { CompressionLevel = K4os.Compression.LZ4.LZ4Level.L04_HC };
31+
32+
normal = SerializeMemoryPack();
33+
brotliFast = BrotliCompressQ1();
34+
lz4Fast = LZ4CompressStreamFast();
35+
}
36+
37+
[Benchmark(Baseline = true), BenchmarkCategory(Categories.Serialize)]
38+
public byte[] SerializeMemoryPack()
39+
{
40+
return MemoryPackSerializer.Serialize(value, MemoryPackSerializeOptions.Utf8);
41+
}
42+
43+
[Benchmark]
44+
public byte[] SerializeMemoryPackUtf16()
45+
{
46+
return MemoryPackSerializer.Serialize(value, MemoryPackSerializeOptions.Utf16);
47+
}
48+
49+
[Benchmark, BenchmarkCategory(Categories.Serialize)]
50+
public byte[] BrotliCompressQ1()
51+
{
52+
using var compressor = new BrotliCompressor(quality: 1);
53+
MemoryPackSerializer.Serialize(compressor, value, MemoryPackSerializeOptions.Utf8);
54+
return compressor.ToArray();
55+
}
56+
57+
[Benchmark, BenchmarkCategory(Categories.Serialize)]
58+
public byte[] BrotliCompressQ2()
59+
{
60+
using var compressor = new BrotliCompressor(quality: 2);
61+
MemoryPackSerializer.Serialize(compressor, value, MemoryPackSerializeOptions.Utf8);
62+
return compressor.ToArray();
63+
}
64+
65+
[Benchmark, BenchmarkCategory(Categories.Serialize)]
66+
public byte[] BrotliCompressQ3()
67+
{
68+
using var compressor = new BrotliCompressor(quality: 3);
69+
MemoryPackSerializer.Serialize(compressor, value, MemoryPackSerializeOptions.Utf8);
70+
return compressor.ToArray();
71+
}
72+
73+
[Benchmark, BenchmarkCategory(Categories.Serialize)]
74+
public byte[] BrotliCompressQ4()
75+
{
76+
using var compressor = new BrotliCompressor(quality: 4);
77+
MemoryPackSerializer.Serialize(compressor, value, MemoryPackSerializeOptions.Utf8);
78+
return compressor.ToArray();
79+
}
80+
81+
[Benchmark, BenchmarkCategory(Categories.Serialize)]
82+
public byte[] BrotliCompressStreamFastest()
83+
{
84+
ms.Position = 0;
85+
using (var brotli = new BrotliStream(ms, CompressionLevel.Fastest, leaveOpen: true))
86+
{
87+
MemoryPackSerializer.SerializeAsync(brotli, value, MemoryPackSerializeOptions.Utf8).ConfigureAwait(false).GetAwaiter().GetResult();
88+
}
89+
ms.Flush();
90+
return ms.ToArray();
91+
}
92+
93+
[Benchmark, BenchmarkCategory(Categories.Serialize)]
94+
public byte[] BrotliCompressStreamOptimial()
95+
{
96+
ms.Position = 0;
97+
using (var brotli = new BrotliStream(ms, CompressionLevel.Optimal, leaveOpen: true))
98+
{
99+
MemoryPackSerializer.SerializeAsync(brotli, value, MemoryPackSerializeOptions.Utf8).ConfigureAwait(false).GetAwaiter().GetResult();
100+
}
101+
ms.Flush();
102+
return ms.ToArray();
103+
}
104+
105+
[Benchmark, BenchmarkCategory(Categories.Serialize)]
106+
public byte[] BrotliCompressStreamSmallestSize()
107+
{
108+
ms.Position = 0;
109+
using (var brotli = new BrotliStream(ms, CompressionLevel.SmallestSize, leaveOpen: true))
110+
{
111+
MemoryPackSerializer.SerializeAsync(brotli, value, MemoryPackSerializeOptions.Utf8).ConfigureAwait(false).GetAwaiter().GetResult();
112+
}
113+
ms.Flush();
114+
return ms.ToArray();
115+
}
116+
117+
[Benchmark, BenchmarkCategory(Categories.Serialize)]
118+
public byte[] BrotliCompressStreamNoCompression()
119+
{
120+
ms.Position = 0;
121+
using (var brotli = new BrotliStream(ms, CompressionLevel.NoCompression, leaveOpen: true))
122+
{
123+
MemoryPackSerializer.SerializeAsync(brotli, value, MemoryPackSerializeOptions.Utf8).ConfigureAwait(false).GetAwaiter().GetResult();
124+
}
125+
ms.Flush();
126+
return ms.ToArray();
127+
}
128+
129+
[Benchmark, BenchmarkCategory(Categories.Serialize)]
130+
public byte[] LZ4CompressStreamFast()
131+
{
132+
133+
134+
ms.Position = 0;
135+
using (var lz4 = LZ4Stream.Encode(ms, Fast, leaveOpen: true))
136+
{
137+
MemoryPackSerializer.SerializeAsync(lz4, value, MemoryPackSerializeOptions.Utf8).ConfigureAwait(false).GetAwaiter().GetResult();
138+
}
139+
ms.Flush();
140+
return ms.ToArray();
141+
}
142+
143+
[Benchmark, BenchmarkCategory(Categories.Serialize)]
144+
public byte[] LZ4CompressStreamHc04()
145+
{
146+
147+
148+
ms.Position = 0;
149+
using (var lz4 = LZ4Stream.Encode(ms, L04HC, leaveOpen: true))
150+
{
151+
MemoryPackSerializer.SerializeAsync(lz4, value, MemoryPackSerializeOptions.Utf8).ConfigureAwait(false).GetAwaiter().GetResult();
152+
}
153+
ms.Flush();
154+
return ms.ToArray();
155+
}
156+
157+
[Benchmark, BenchmarkCategory(Categories.Serialize)]
158+
public byte[] LZ4CompressStreamL10Opt()
159+
{
160+
ms.Position = 0;
161+
using (var lz4 = LZ4Stream.Encode(ms, L10Opt, leaveOpen: true))
162+
{
163+
MemoryPackSerializer.SerializeAsync(lz4, value, MemoryPackSerializeOptions.Utf8).ConfigureAwait(false).GetAwaiter().GetResult();
164+
}
165+
ms.Flush();
166+
return ms.ToArray();
167+
}
168+
169+
//Decompress
170+
171+
[Benchmark(Baseline = true), BenchmarkCategory(Categories.Deserialize)]
172+
public T? DeserializeMemoryPack()
173+
{
174+
return MemoryPackSerializer.Deserialize<T>(normal);
175+
}
176+
177+
[Benchmark(), BenchmarkCategory(Categories.Deserialize)]
178+
public T? DecompressBrotli()
179+
{
180+
using var decompressor = new BrotliDecompressor();
181+
return MemoryPackSerializer.Deserialize<T>(decompressor.Decompress(brotliFast));
182+
}
183+
184+
[Benchmark(), BenchmarkCategory(Categories.Deserialize)]
185+
public T? DecompressBrotliStream()
186+
{
187+
using (var ms2 = new MemoryStream(brotliFast))
188+
using (var brotli = new BrotliStream(ms2, CompressionMode.Decompress))
189+
{
190+
return MemoryPackSerializer.DeserializeAsync<T>(brotli).GetAwaiter().GetResult();
191+
}
192+
}
193+
194+
[Benchmark(), BenchmarkCategory(Categories.Deserialize)]
195+
public T? LZ4Decompress()
196+
{
197+
using (var ms2 = new MemoryStream(lz4Fast))
198+
using (var lz4 = LZ4Stream.Decode(ms2, leaveOpen: true))
199+
{
200+
return MemoryPackSerializer.DeserializeAsync<T>(lz4).GetAwaiter().GetResult();
201+
}
202+
}
203+
204+
// GZip
205+
206+
207+
208+
//[Benchmark]
209+
//public byte[] GZipCompressStreamFastest()
210+
//{
211+
// ms.Position = 0;
212+
// using (var GZip = new GZipStream(ms, CompressionLevel.Fastest, leaveOpen: true))
213+
// {
214+
// MemoryPackSerializer.SerializeAsync(GZip, value, MemoryPackSerializeOptions.Utf8).ConfigureAwait(false).GetAwaiter().GetResult();
215+
// }
216+
// ms.Flush();
217+
// return ms.ToArray();
218+
//}
219+
220+
//[Benchmark]
221+
//public byte[] GZipCompressStreamOptimial()
222+
//{
223+
// ms.Position = 0;
224+
// using (var GZip = new GZipStream(ms, CompressionLevel.Optimal, leaveOpen: true))
225+
// {
226+
// MemoryPackSerializer.SerializeAsync(GZip, value, MemoryPackSerializeOptions.Utf8).ConfigureAwait(false).GetAwaiter().GetResult();
227+
// }
228+
// ms.Flush();
229+
// return ms.ToArray();
230+
//}
231+
232+
//[Benchmark]
233+
//public byte[] GZipCompressStreamSmallestSize()
234+
//{
235+
// ms.Position = 0;
236+
// using (var GZip = new GZipStream(ms, CompressionLevel.SmallestSize, leaveOpen: true))
237+
// {
238+
// MemoryPackSerializer.SerializeAsync(GZip, value, MemoryPackSerializeOptions.Utf8).ConfigureAwait(false).GetAwaiter().GetResult();
239+
// }
240+
// ms.Flush();
241+
// return ms.ToArray();
242+
//}
243+
244+
//[Benchmark]
245+
//public byte[] GZipCompressStreamNoCompression()
246+
//{
247+
// ms.Position = 0;
248+
// using (var GZip = new GZipStream(ms, CompressionLevel.NoCompression, leaveOpen: true))
249+
// {
250+
// MemoryPackSerializer.SerializeAsync(GZip, value, MemoryPackSerializeOptions.Utf8).ConfigureAwait(false).GetAwaiter().GetResult();
251+
// }
252+
// ms.Flush();
253+
// return ms.ToArray();
254+
//}
255+
256+
}

‎sandbox/Benchmark/Program.cs

+17-3
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
using BinaryPack.Models.Interfaces;
1212
using Iced.Intel;
1313
using MemoryPack;
14+
using MemoryPack.Compression;
1415
using MemoryPack.Formatters;
1516
using System.Reflection;
1617

@@ -51,7 +52,12 @@
5152
// BenchmarkRunner.Run<DeserializeTest<NeuralNetworkLayerModel>>(config, args);
5253

5354

54-
BenchmarkRunner.Run<DeserializeTest<JsonResponseModel>>(config, args);
55+
//BenchmarkRunner.Run<DeserializeTest<JsonResponseModel>>(config, args);
56+
57+
58+
BenchmarkRunner.Run<Compression<JsonResponseModel>>(config, args);
59+
//BenchmarkRunner.Run<Compression<Vector3[]>>(config, args);
60+
//BenchmarkRunner.Run<Compression<NeuralNetworkLayerModel>>(config, args);
5561

5662

5763
//BenchmarkRunner.Run<GetLocalVsStaticField>(config, args);
@@ -65,8 +71,16 @@
6571

6672
#if DEBUG
6773

68-
var foo = new Utf8Decoding().Utf16LengthUtf8ToUtf16();
69-
Console.WriteLine(foo);
74+
var model = new JsonResponseModel(true);
75+
var model2 = Enumerable.Repeat(new Vector3 { X = 10.3f, Y = 40.5f, Z = 13411.3f }, 1000).ToArray();
76+
77+
using var compressor = new BrotliCompressor();
78+
MemoryPackSerializer.Serialize(compressor, model2);
79+
var foo = compressor.ToArray();
80+
81+
using var decompressor = new BrotliDecompressor();
82+
83+
var foo2 = decompressor.Decompress(foo);
7084

7185
Check<JsonResponseModel>();
7286
Check<NeuralNetworkLayerModel>();

0 commit comments

Comments
 (0)