From f87b62a483d3004f5efe70807db9f421c6c0677c Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 4 Aug 2021 18:04:00 +0200 Subject: [PATCH 1/5] skip AwaitingTasksShouldNotInterfereAllocationResults for InProcessToolchain (it's unstable) --- tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 89af9fd4c9..25d17208ad 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -147,6 +147,9 @@ public class NonAllocatingAsynchronousBenchmarks [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AwaitingTasksShouldNotInterfereAllocationResults(IToolchain toolchain) { + if (toolchain is InProcessEmitToolchain) // this test is unstable for InProcess toolchain + return; + AssertAllocations(toolchain, typeof(NonAllocatingAsynchronousBenchmarks), new Dictionary { { nameof(NonAllocatingAsynchronousBenchmarks.CompletedTask), 0 }, From 2eb9e73b19c948e9ff57a5a195007ddf9911bb0b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 4 Aug 2021 19:07:19 +0200 Subject: [PATCH 2/5] play with the tests --- .../MemoryDiagnoserTests.cs | 16 ++++++---------- .../XUnit/TheoryNetCore30Attribute.cs | 16 ---------------- 2 files changed, 6 insertions(+), 26 deletions(-) delete mode 100644 tests/BenchmarkDotNet.Tests/XUnit/TheoryNetCore30Attribute.cs diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 25d17208ad..87da175b50 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -37,8 +37,9 @@ public static IEnumerable GetToolchains() : new[] { new object[] { Job.Default.GetToolchain() }, - new object[] { InProcessEmitToolchain.Instance }, -#if !NETFRAMEWORK +#if NETFRAMEWORK + new object[] { InProcessEmitToolchain.Instance }, // it seems to be unstable for .NET 5 +#else // we don't want to test CoreRT twice (for .NET 4.6 and 5.0) when running the integration tests (these tests take a lot of time) // we test against specific version to keep this test stable new object[] { CoreRtToolchain.CreateBuilder().UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "6.0.0-preview.1.21074.3").ToToolchain() } @@ -92,7 +93,7 @@ private void AllocateUntilGcWakesUp() } } - [Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolchain toolchain) { @@ -147,9 +148,6 @@ public class NonAllocatingAsynchronousBenchmarks [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AwaitingTasksShouldNotInterfereAllocationResults(IToolchain toolchain) { - if (toolchain is InProcessEmitToolchain) // this test is unstable for InProcess toolchain - return; - AssertAllocations(toolchain, typeof(NonAllocatingAsynchronousBenchmarks), new Dictionary { { nameof(NonAllocatingAsynchronousBenchmarks.CompletedTask), 0 }, @@ -197,8 +195,7 @@ public byte[] SixtyFourBytesArray() } } - [Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))] - //[TheoryNetCoreOnly("Only .NET Core 2.0+ API is bug free for this case"), MemberData(nameof(GetToolchains))] + [TheoryNetCoreOnly("Only .NET Core 2.0+ API is bug free for this case"), MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { @@ -240,7 +237,7 @@ public void Allocate() } } - [TheoryNetCore30(".NET Core 3.0 preview6+ exposes a GC.GetTotalAllocatedBytes method which makes it possible to work"), MemberData(nameof(GetToolchains))] + [TheoryNetCoreOnly(".NET Core 3.0 preview6+ exposes a GC.GetTotalAllocatedBytes method which makes it possible to work"), MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain) { @@ -296,7 +293,6 @@ private IConfig CreateConfig(IToolchain toolchain) .WithWarmupCount(0) // don't run warmup to save some time for our CI runs .WithIterationCount(1) // single iteration is enough for us .WithGcForce(false) - .WithEnvironmentVariable("COMPlus_TieredCompilation", "0") // Tiered JIT can allocate some memory on a background thread, let's disable it to make our tests less flaky (#1542) .WithToolchain(toolchain)) .AddColumnProvider(DefaultColumnProviders.Instance) .AddDiagnoser(MemoryDiagnoser.Default) diff --git a/tests/BenchmarkDotNet.Tests/XUnit/TheoryNetCore30Attribute.cs b/tests/BenchmarkDotNet.Tests/XUnit/TheoryNetCore30Attribute.cs deleted file mode 100644 index c91de5170a..0000000000 --- a/tests/BenchmarkDotNet.Tests/XUnit/TheoryNetCore30Attribute.cs +++ /dev/null @@ -1,16 +0,0 @@ -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Portability; -using Xunit; - -namespace BenchmarkDotNet.Tests.XUnit -{ - public class TheoryNetCore30Attribute : TheoryAttribute - { - // ReSharper disable once VirtualMemberCallInConstructor - public TheoryNetCore30Attribute(string skipReason) - { - if (RuntimeInformation.GetCurrentRuntime().RuntimeMoniker != RuntimeMoniker.NetCoreApp30) - Skip = skipReason; - } - } -} \ No newline at end of file From 3b2d7369884d591e27da5bb2e2cb16221cc3945b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 4 Aug 2021 20:12:14 +0200 Subject: [PATCH 3/5] reduce the number of test cases (1 should be enough) --- .../MemoryDiagnoserTests.cs | 46 +------------------ 1 file changed, 2 insertions(+), 44 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 87da175b50..bac2ad983e 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -48,10 +48,7 @@ public static IEnumerable GetToolchains() public class AccurateAllocations { - [Benchmark] public byte[] EightBytesArray() => new byte[8]; - [Benchmark] public byte[] SixtyFourBytesArray() => new byte[64]; - - [Benchmark] public Task AllocateTask() => Task.FromResult(default(int)); + [Benchmark] public byte[] SmallObjectHeap() => new byte[8]; } [Theory, MemberData(nameof(GetToolchains))] @@ -63,10 +60,7 @@ public void MemoryDiagnoserIsAccurate(IToolchain toolchain) AssertAllocations(toolchain, typeof(AccurateAllocations), new Dictionary { - { nameof(AccurateAllocations.EightBytesArray), 8 + objectAllocationOverhead + arraySizeOverhead }, - { nameof(AccurateAllocations.SixtyFourBytesArray), 64 + objectAllocationOverhead + arraySizeOverhead }, - - { nameof(AccurateAllocations.AllocateTask), CalculateRequiredSpace>() }, + { nameof(AccurateAllocations.SmallObjectHeap), 8 + objectAllocationOverhead + arraySizeOverhead }, }); } @@ -265,11 +259,6 @@ private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Diction foreach (var benchmarkAllocationsValidator in benchmarksAllocationsValidators) { - // CoreRT is missing some of the CoreCLR threading/task related perf improvements, so sizeof(Task) calculated for CoreCLR < sizeof(Task) on CoreRT - // see https://github.com/dotnet/corert/issues/5705 for more - if (benchmarkAllocationsValidator.Key == nameof(AccurateAllocations.AllocateTask) && toolchain is CoreRtToolchain) - continue; - var allocatingBenchmarks = benchmarks.BenchmarksCases.Where(benchmark => benchmark.DisplayInfo.Contains(benchmarkAllocationsValidator.Key)); foreach (var benchmark in allocatingBenchmarks) @@ -297,36 +286,5 @@ private IConfig CreateConfig(IToolchain toolchain) .AddColumnProvider(DefaultColumnProviders.Instance) .AddDiagnoser(MemoryDiagnoser.Default) .AddLogger(toolchain.IsInProcess ? ConsoleLogger.Default : new OutputLogger(output)); // we can't use OutputLogger for the InProcess toolchains because it allocates memory on the same thread - - // note: don't copy, never use in production systems (it should work but I am not 100% sure) - private int CalculateRequiredSpace() - { - int total = SizeOfAllFields(); - - if (!typeof(T).GetTypeInfo().IsValueType) - total += IntPtr.Size * 2; // pointer to method table + object header word - - if (total % IntPtr.Size != 0) // aligning.. - total += IntPtr.Size - (total % IntPtr.Size); - - return total; - } - - // note: don't copy, never use in production systems (it should work but I am not 100% sure) - private int SizeOfAllFields() - { - int GetSize(Type type) - { - var sizeOf = typeof(Unsafe).GetTypeInfo().GetMethod(nameof(Unsafe.SizeOf)); - - return (int)sizeOf.MakeGenericMethod(type).Invoke(null, null); - } - - return typeof(T) - .GetAllFields() - .Where(field => !field.IsStatic && !field.IsLiteral) - .Distinct() - .Sum(field => GetSize(field.FieldType)); - } } } From 0e57b77ace853e6b5811f099fc7b121ed8d5273a Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 4 Aug 2021 20:27:09 +0200 Subject: [PATCH 4/5] these tests are passing on CoreRT now --- .../MemoryDiagnoserTests.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index bac2ad983e..43554b2236 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -193,9 +193,6 @@ public byte[] SixtyFourBytesArray() [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { - if (toolchain is CoreRtToolchain) // the fix has not yet been backported to CoreRT - return; - long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length @@ -235,13 +232,10 @@ public void Allocate() [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain) { - if (toolchain is CoreRtToolchain) // the API has not been yet ported to CoreRT - return; - long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length long memoryAllocatedPerArray = (MultiThreadedAllocation.Size + objectAllocationOverhead + arraySizeOverhead); - long threadStartAndJoinOverhead = 112; // this is more or less a magic number taken from memory profiler + long threadStartAndJoinOverhead = toolchain is CoreRtToolchain ? 560 : 112; // this is more or less a magic number taken from memory profiler long allocatedMemoryPerThread = memoryAllocatedPerArray + threadStartAndJoinOverhead; AssertAllocations(toolchain, typeof(MultiThreadedAllocation), new Dictionary From a2e930f3db70e51018752ada0da88f6d259a9c55 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 5 Aug 2021 10:34:36 +0200 Subject: [PATCH 5/5] use ranges for MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks test --- .../MemoryDiagnoserTests.cs | 49 ++++++++++++------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 43554b2236..4f2ce58463 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -9,6 +9,7 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; @@ -228,44 +229,56 @@ public void Allocate() } } - [TheoryNetCoreOnly(".NET Core 3.0 preview6+ exposes a GC.GetTotalAllocatedBytes method which makes it possible to work"), MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain) { long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length long memoryAllocatedPerArray = (MultiThreadedAllocation.Size + objectAllocationOverhead + arraySizeOverhead); - long threadStartAndJoinOverhead = toolchain is CoreRtToolchain ? 560 : 112; // this is more or less a magic number taken from memory profiler - long allocatedMemoryPerThread = memoryAllocatedPerArray + threadStartAndJoinOverhead; + long maxThreadStartAndJoinOverhead = RuntimeInformation.IsFullFramework ? 2500 : 600; // these are more or less a magic numbers taken from memory profiler and test runs - AssertAllocations(toolchain, typeof(MultiThreadedAllocation), new Dictionary + AssertAllocations(toolchain, typeof(MultiThreadedAllocation), new Dictionary { - { nameof(MultiThreadedAllocation.Allocate), allocatedMemoryPerThread * MultiThreadedAllocation.ThreadsCount } + { nameof(MultiThreadedAllocation.Allocate), ( + low: memoryAllocatedPerArray * MultiThreadedAllocation.ThreadsCount, + high: (memoryAllocatedPerArray + maxThreadStartAndJoinOverhead) * MultiThreadedAllocation.ThreadsCount) } }); } private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators) + { + AssertAllocations(toolchain, benchmarkType, benchmarksAllocationsValidators, (benchmark, gcStats, expectedBytes) => + { + Assert.Equal(expectedBytes, gcStats.GetBytesAllocatedPerOperation(benchmark)); + + if (expectedBytes == 0) + { + Assert.Equal(0, gcStats.GetTotalAllocatedBytes(excludeAllocationQuantumSideEffects: true)); + } + }); + } + + private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators) + { + AssertAllocations(toolchain, benchmarkType, benchmarksAllocationsValidators, (benchmark, gcStats, expectedBytes) => + { + Assert.InRange(gcStats.GetBytesAllocatedPerOperation(benchmark), expectedBytes.low, expectedBytes.high); + }); + } + + private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators, Action assert) { var config = CreateConfig(toolchain); var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config); var summary = BenchmarkRunner.Run(benchmarks); - foreach (var benchmarkAllocationsValidator in benchmarksAllocationsValidators) + foreach (var report in summary.Reports) { - var allocatingBenchmarks = benchmarks.BenchmarksCases.Where(benchmark => benchmark.DisplayInfo.Contains(benchmarkAllocationsValidator.Key)); + var benchmark = report.BenchmarkCase; - foreach (var benchmark in allocatingBenchmarks) - { - var benchmarkReport = summary.Reports.Single(report => report.BenchmarkCase == benchmark); - - Assert.Equal(benchmarkAllocationsValidator.Value, benchmarkReport.GcStats.GetBytesAllocatedPerOperation(benchmark)); - - if (benchmarkAllocationsValidator.Value == 0) - { - Assert.Equal(0, benchmarkReport.GcStats.GetTotalAllocatedBytes(excludeAllocationQuantumSideEffects: true)); - } - } + assert(benchmark, report.GcStats, benchmarksAllocationsValidators[benchmark.Descriptor.WorkloadMethod.Name]); } }