Skip to content

Commit e8a8fc6

Browse files
adamsitnikadamsitnik
authored
Refactor initialization logic to allow for enabling Memory Randomization (#1587)
* there is no need for the warmup workaround anymore, dotnet/BenchmarkDotNet#1573 has solved the problem * allocate the array first to try to take advantage of memory randomization as it's usually the first thing called from GlobalSetup method which with MemoryRandomization enabled is the first method called right after allocation of random-sized memory by BDN engine * avoid having big global setup methods and try to have Targeted setups that allocate less as Global Setup methods might get called right after random-size memory allocation * avoid readonly fields, initialize them in [GlobalSetup] instead of ctors (to allow for re-allocation with different alignment) * this code can be executed only once * make sure that every setup creates a brand new delegate instead of combining with existing one * the "WriteDeepUtf16" benchmark can report up to x4 more time when MemoryRandmization is enabled this is due to having new _arrayBufferWriter every time and allocating a lot of memory so we don't always allocate a new instance * allocate the array in GlobalSetup, not in field initializer
1 parent ed8fbe6 commit e8a8fc6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+632
-486
lines changed

‎src/benchmarks/micro/Serializers/Binary_FromStream.cs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,13 @@ public T BinaryFormatter_()
4545
public void SetupProtoBuffNet()
4646
{
4747
value = DataGenerator.Generate<T>();
48+
if (memoryStream is null) // to ensure it's done only once
49+
{
50+
ProtoBuf.Meta.RuntimeTypeModel.Default.Add(typeof(DateTimeOffset), false).SetSurrogate(typeof(DateTimeOffsetSurrogate)); // https://stackoverflow.com/a/7046868
51+
}
4852
// the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
4953
memoryStream = new MemoryStream(capacity: short.MaxValue);
5054
memoryStream.Position = 0;
51-
ProtoBuf.Meta.RuntimeTypeModel.Default.Add(typeof(DateTimeOffset), false).SetSurrogate(typeof(DateTimeOffsetSurrogate)); // https://stackoverflow.com/a/7046868
5255
ProtoBuf.Serializer.Serialize(memoryStream, value);
5356
}
5457

‎src/benchmarks/micro/Serializers/Json_FromStream.cs

Lines changed: 45 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -16,74 +16,55 @@ namespace MicroBenchmarks.Serializers
1616
[GenericTypeArguments(typeof(CollectionsOfPrimitives))]
1717
public class Json_FromStream<T>
1818
{
19-
private readonly T value;
20-
21-
private readonly MemoryStream memoryStream;
22-
19+
private T value;
20+
private MemoryStream memoryStream;
2321
private DataContractJsonSerializer dataContractJsonSerializer;
2422
private Newtonsoft.Json.JsonSerializer newtonSoftJsonSerializer;
2523

26-
public Json_FromStream()
24+
[GlobalSetup(Target = nameof(Jil_))]
25+
public void SetupJil_()
2726
{
2827
value = DataGenerator.Generate<T>();
2928

3029
// the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
3130
memoryStream = new MemoryStream(capacity: short.MaxValue);
32-
33-
dataContractJsonSerializer = new DataContractJsonSerializer(typeof(T));
34-
newtonSoftJsonSerializer = new Newtonsoft.Json.JsonSerializer();
35-
}
36-
37-
[GlobalSetup(Target = nameof(Jil_))]
38-
public void SetupJil_()
39-
{
4031
memoryStream.Position = 0;
4132

4233
using (var writer = new StreamWriter(memoryStream, Encoding.UTF8, short.MaxValue, leaveOpen: true))
4334
{
4435
Jil.JSON.Serialize<T>(value, writer, Jil.Options.ISO8601);
4536
writer.Flush();
4637
}
38+
}
39+
40+
[BenchmarkCategory(Categories.ThirdParty)]
41+
[Benchmark(Description = "Jil")]
42+
public T Jil_()
43+
{
44+
memoryStream.Position = 0;
4745

48-
Jil_(); // workaround for https://github.com/dotnet/BenchmarkDotNet/issues/837
46+
using (var reader = CreateNonClosingReaderWithDefaultSizes())
47+
return Jil.JSON.Deserialize<T>(reader, Jil.Options.ISO8601);
4948
}
5049

5150
[GlobalSetup(Target = nameof(JsonNet_))]
5251
public void SetupJsonNet_()
5352
{
53+
value = DataGenerator.Generate<T>();
54+
55+
// the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
56+
memoryStream = new MemoryStream(capacity: short.MaxValue);
5457
memoryStream.Position = 0;
5558

59+
newtonSoftJsonSerializer = new Newtonsoft.Json.JsonSerializer();
60+
5661
using (var writer = new StreamWriter(memoryStream, Encoding.UTF8, short.MaxValue, leaveOpen: true))
5762
{
5863
newtonSoftJsonSerializer.Serialize(writer, value);
5964
writer.Flush();
6065
}
6166
}
6267

63-
[GlobalSetup(Target = nameof(Utf8Json_))]
64-
public void SetupUtf8Json_()
65-
{
66-
memoryStream.Position = 0;
67-
Utf8Json.JsonSerializer.Serialize<T>(memoryStream, value);
68-
}
69-
70-
[GlobalSetup(Target = nameof(DataContractJsonSerializer_))]
71-
public void SetupDataContractJsonSerializer_()
72-
{
73-
memoryStream.Position = 0;
74-
dataContractJsonSerializer.WriteObject(memoryStream, value);
75-
}
76-
77-
[BenchmarkCategory(Categories.ThirdParty)]
78-
[Benchmark(Description = "Jil")]
79-
public T Jil_()
80-
{
81-
memoryStream.Position = 0;
82-
83-
using (var reader = CreateNonClosingReaderWithDefaultSizes())
84-
return Jil.JSON.Deserialize<T>(reader, Jil.Options.ISO8601);
85-
}
86-
8768
[BenchmarkCategory(Categories.Runtime, Categories.Libraries, Categories.ThirdParty)] // JSON.NET is so popular that despite being 3rd Party lib we run the benchmarks for CoreFX and CoreCLR CI
8869
[Benchmark(Description = "JSON.NET")]
8970
public T JsonNet_()
@@ -94,6 +75,17 @@ public T JsonNet_()
9475
return (T)newtonSoftJsonSerializer.Deserialize(reader, typeof(T));
9576
}
9677

78+
[GlobalSetup(Target = nameof(Utf8Json_))]
79+
public void SetupUtf8Json_()
80+
{
81+
value = DataGenerator.Generate<T>();
82+
83+
// the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
84+
memoryStream = new MemoryStream(capacity: short.MaxValue);
85+
memoryStream.Position = 0;
86+
Utf8Json.JsonSerializer.Serialize<T>(memoryStream, value);
87+
}
88+
9789
[BenchmarkCategory(Categories.ThirdParty)]
9890
[Benchmark(Description = "Utf8Json")]
9991
public T Utf8Json_()
@@ -102,6 +94,18 @@ public T Utf8Json_()
10294
return Utf8Json.JsonSerializer.Deserialize<T>(memoryStream);
10395
}
10496

97+
[GlobalSetup(Target = nameof(DataContractJsonSerializer_))]
98+
public void SetupDataContractJsonSerializer_()
99+
{
100+
value = DataGenerator.Generate<T>();
101+
102+
// the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
103+
memoryStream = new MemoryStream(capacity: short.MaxValue);
104+
memoryStream.Position = 0;
105+
dataContractJsonSerializer = new DataContractJsonSerializer(typeof(T));
106+
dataContractJsonSerializer.WriteObject(memoryStream, value);
107+
}
108+
105109
[BenchmarkCategory(Categories.Runtime, Categories.Libraries)]
106110
[Benchmark(Description = "DataContractJsonSerializer")]
107111
public T DataContractJsonSerializer_()
@@ -110,15 +114,15 @@ public T DataContractJsonSerializer_()
110114
return (T)dataContractJsonSerializer.ReadObject(memoryStream);
111115
}
112116

117+
[GlobalCleanup]
118+
public void Cleanup() => memoryStream.Dispose();
119+
113120
private StreamReader CreateNonClosingReaderWithDefaultSizes()
114121
=> new StreamReader(
115122
memoryStream,
116123
Encoding.UTF8,
117124
true, // default is true https://github.com/dotnet/corefx/blob/708e4537d8944199af7d580def0d97a030be98c7/src/Common/src/CoreLib/System/IO/StreamReader.cs#L98
118125
1024, // default buffer size from CoreFX https://github.com/dotnet/corefx/blob/708e4537d8944199af7d580def0d97a030be98c7/src/Common/src/CoreLib/System/IO/StreamReader.cs#L27
119126
leaveOpen: true); // we want to reuse the same string in the benchmarks to make sure that cost of allocating stream is not included in the benchmarks
120-
121-
[GlobalCleanup]
122-
public void Cleanup() => memoryStream.Dispose();
123127
}
124128
}

‎src/benchmarks/micro/Serializers/Json_FromString.cs

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,33 +13,25 @@ namespace MicroBenchmarks.Serializers
1313
[GenericTypeArguments(typeof(CollectionsOfPrimitives))]
1414
public class Json_FromString<T>
1515
{
16-
private readonly T value;
1716
private string serialized;
1817

19-
public Json_FromString() => value = DataGenerator.Generate<T>();
20-
2118
[GlobalSetup(Target = nameof(Jil_))]
22-
public void SerializeJil()
23-
{
24-
serialized = Jil.JSON.Serialize<T>(value, Jil.Options.ISO8601);
25-
26-
Jil_(); // workaround for https://github.com/dotnet/BenchmarkDotNet/issues/837
27-
}
28-
29-
[GlobalSetup(Target = nameof(JsonNet_))]
30-
public void SerializeJsonNet() => serialized = Newtonsoft.Json.JsonConvert.SerializeObject(value);
31-
32-
[GlobalSetup(Target = nameof(Utf8Json_))]
33-
public void SerializeUtf8Json_() => serialized = Utf8Json.JsonSerializer.ToJsonString(value);
19+
public void SetupJil() => serialized = Jil.JSON.Serialize<T>(DataGenerator.Generate<T>(), Jil.Options.ISO8601);
3420

3521
[BenchmarkCategory(Categories.ThirdParty)]
3622
[Benchmark(Description = "Jil")]
3723
public T Jil_() => Jil.JSON.Deserialize<T>(serialized, Jil.Options.ISO8601);
3824

25+
[GlobalSetup(Target = nameof(JsonNet_))]
26+
public void SerializeJsonNet() => serialized = Newtonsoft.Json.JsonConvert.SerializeObject(DataGenerator.Generate<T>());
27+
3928
[BenchmarkCategory(Categories.Runtime, Categories.Libraries, Categories.ThirdParty)]
4029
[Benchmark(Description = "JSON.NET")]
4130
public T JsonNet_() => Newtonsoft.Json.JsonConvert.DeserializeObject<T>(serialized);
4231

32+
[GlobalSetup(Target = nameof(Utf8Json_))]
33+
public void SerializeUtf8Json_() => serialized = Utf8Json.JsonSerializer.ToJsonString(DataGenerator.Generate<T>());
34+
4335
[BenchmarkCategory(Categories.ThirdParty)]
4436
[Benchmark(Description = "Utf8Json")]
4537
public T Utf8Json_() => Utf8Json.JsonSerializer.Deserialize<T>(serialized);

‎src/benchmarks/micro/Serializers/Json_ToStream.cs

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,16 @@ namespace MicroBenchmarks.Serializers
1616
[GenericTypeArguments(typeof(CollectionsOfPrimitives))]
1717
public class Json_ToStream<T>
1818
{
19-
private readonly T value;
19+
private T value;
2020

21-
private readonly MemoryStream memoryStream;
22-
private readonly StreamWriter streamWriter;
21+
private MemoryStream memoryStream;
22+
private StreamWriter streamWriter;
2323

2424
private DataContractJsonSerializer dataContractJsonSerializer;
2525
private Newtonsoft.Json.JsonSerializer newtonSoftJsonSerializer;
2626

27-
public Json_ToStream()
27+
[GlobalSetup]
28+
public void Setup()
2829
{
2930
value = DataGenerator.Generate<T>();
3031

@@ -36,9 +37,6 @@ public Json_ToStream()
3637
newtonSoftJsonSerializer = new Newtonsoft.Json.JsonSerializer();
3738
}
3839

39-
[GlobalSetup(Target = nameof(Jil_))]
40-
public void WarmupJil() => Jil_(); // workaround for https://github.com/dotnet/BenchmarkDotNet/issues/837
41-
4240
[BenchmarkCategory(Categories.ThirdParty)]
4341
[Benchmark(Description = "Jil")]
4442
public void Jil_()

‎src/benchmarks/micro/Serializers/Json_ToString.cs

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,10 @@ namespace MicroBenchmarks.Serializers
1313
[GenericTypeArguments(typeof(CollectionsOfPrimitives))]
1414
public class Json_ToString<T>
1515
{
16-
private readonly T value;
16+
private T value;
1717

18-
public Json_ToString() => value = DataGenerator.Generate<T>();
19-
20-
[GlobalSetup(Target = nameof(Jil_))]
21-
public void WarmupJil() => Jil_(); // workaround for https://github.com/dotnet/BenchmarkDotNet/issues/837
18+
[GlobalSetup]
19+
public void Setup() => value = DataGenerator.Generate<T>();
2220

2321
[BenchmarkCategory(Categories.ThirdParty)]
2422
[Benchmark(Description = "Jil")]

‎src/benchmarks/micro/Serializers/Xml_FromStream.cs

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,41 +20,39 @@ namespace MicroBenchmarks.Serializers
2020
[GenericTypeArguments(typeof(ClassImplementingIXmlSerialiable))]
2121
public class Xml_FromStream<T>
2222
{
23-
private readonly T value;
24-
private readonly XmlSerializer xmlSerializer;
25-
private readonly DataContractSerializer dataContractSerializer;
26-
private readonly MemoryStream memoryStream;
23+
private T value;
24+
private XmlSerializer xmlSerializer;
25+
private DataContractSerializer dataContractSerializer;
26+
private MemoryStream memoryStream;
2727

28-
public Xml_FromStream()
28+
[GlobalSetup(Target = nameof(XmlSerializer_))]
29+
public void SetupXmlSerializer()
2930
{
3031
value = DataGenerator.Generate<T>();
31-
xmlSerializer = new XmlSerializer(typeof(T));
32-
dataContractSerializer = new DataContractSerializer(typeof(T));
3332
memoryStream = new MemoryStream(capacity: short.MaxValue);
33+
memoryStream.Position = 0;
34+
xmlSerializer = new XmlSerializer(typeof(T));
35+
xmlSerializer.Serialize(memoryStream, value);
3436
}
3537

36-
[GlobalSetup(Target = nameof(XmlSerializer_))]
37-
public void SetupXmlSerializer()
38+
[BenchmarkCategory(Categories.Libraries, Categories.Runtime)]
39+
[Benchmark(Description = nameof(XmlSerializer))]
40+
public T XmlSerializer_()
3841
{
3942
memoryStream.Position = 0;
40-
xmlSerializer.Serialize(memoryStream, value);
43+
return (T)xmlSerializer.Deserialize(memoryStream);
4144
}
4245

4346
[GlobalSetup(Target = nameof(DataContractSerializer_))]
4447
public void SetupDataContractSerializer()
4548
{
49+
value = DataGenerator.Generate<T>();
50+
memoryStream = new MemoryStream(capacity: short.MaxValue);
4651
memoryStream.Position = 0;
52+
dataContractSerializer = new DataContractSerializer(typeof(T));
4753
dataContractSerializer.WriteObject(memoryStream, value);
4854
}
4955

50-
[BenchmarkCategory(Categories.Libraries, Categories.Runtime)]
51-
[Benchmark(Description = nameof(XmlSerializer))]
52-
public T XmlSerializer_()
53-
{
54-
memoryStream.Position = 0;
55-
return (T)xmlSerializer.Deserialize(memoryStream);
56-
}
57-
5856
[BenchmarkCategory(Categories.Libraries)]
5957
[Benchmark(Description = nameof(DataContractSerializer))]
6058
public T DataContractSerializer_()

‎src/benchmarks/micro/Serializers/Xml_ToStream.cs

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,18 @@ namespace MicroBenchmarks.Serializers
2020
[GenericTypeArguments(typeof(ClassImplementingIXmlSerialiable))]
2121
public class Xml_ToStream<T>
2222
{
23-
private readonly T value;
24-
private readonly XmlSerializer xmlSerializer;
25-
private readonly DataContractSerializer dataContractSerializer;
26-
private readonly MemoryStream memoryStream;
23+
private T value;
24+
private XmlSerializer xmlSerializer;
25+
private DataContractSerializer dataContractSerializer;
26+
private MemoryStream memoryStream;
2727

28-
public Xml_ToStream()
28+
[GlobalSetup]
29+
public void Setup()
2930
{
3031
value = DataGenerator.Generate<T>();
32+
memoryStream = new MemoryStream(capacity: short.MaxValue);
3133
xmlSerializer = new XmlSerializer(typeof(T));
3234
dataContractSerializer = new DataContractSerializer(typeof(T));
33-
memoryStream = new MemoryStream(capacity: short.MaxValue);
3435
}
3536

3637
[BenchmarkCategory(Categories.Libraries, Categories.Runtime)]

‎src/benchmarks/micro/libraries/Microsoft.Extensions.Logging/EventSourceBenchmark.cs

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using System.Collections.Generic;
65
using System.Diagnostics.Tracing;
76
using BenchmarkDotNet.Attributes;
87
using Microsoft.Extensions.DependencyInjection;
@@ -59,19 +58,13 @@ public void Setup()
5958
}
6059

6160
[GlobalCleanup]
62-
public void Cleanup()
63-
{
64-
_listener?.Dispose();
65-
}
61+
public void Cleanup() => _listener?.Dispose();
6662

6763
private class TestEventListener : EventListener
6864
{
6965
private readonly EventKeywords _keywords;
7066

71-
public TestEventListener(EventKeywords keywords)
72-
{
73-
_keywords = keywords;
74-
}
67+
public TestEventListener(EventKeywords keywords) => _keywords = keywords;
7568

7669
protected override void OnEventSourceCreated(System.Diagnostics.Tracing.EventSource eventSource)
7770
{

‎src/benchmarks/micro/libraries/Microsoft.Extensions.Logging/LoggingOverheadBenchmark.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using System;
65
using BenchmarkDotNet.Attributes;
76
using Microsoft.Extensions.DependencyInjection;
87
using MicroBenchmarks;

0 commit comments

Comments
 (0)