diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 89af9fd4c9..4f2ce58463 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -9,6 +9,7 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; @@ -37,8 +38,9 @@ public static IEnumerable GetToolchains() : new[] { new object[] { Job.Default.GetToolchain() }, - new object[] { InProcessEmitToolchain.Instance }, -#if !NETFRAMEWORK +#if NETFRAMEWORK + new object[] { InProcessEmitToolchain.Instance }, // it seems to be unstable for .NET 5 +#else // we don't want to test CoreRT twice (for .NET 4.6 and 5.0) when running the integration tests (these tests take a lot of time) // we test against specific version to keep this test stable new object[] { CoreRtToolchain.CreateBuilder().UseCoreRtNuGet(microsoftDotNetILCompilerVersion: "6.0.0-preview.1.21074.3").ToToolchain() } @@ -47,10 +49,7 @@ public static IEnumerable GetToolchains() public class AccurateAllocations { - [Benchmark] public byte[] EightBytesArray() => new byte[8]; - [Benchmark] public byte[] SixtyFourBytesArray() => new byte[64]; - - [Benchmark] public Task AllocateTask() => Task.FromResult(default(int)); + [Benchmark] public byte[] SmallObjectHeap() => new byte[8]; } [Theory, MemberData(nameof(GetToolchains))] @@ -62,10 +61,7 @@ public void MemoryDiagnoserIsAccurate(IToolchain toolchain) AssertAllocations(toolchain, typeof(AccurateAllocations), new Dictionary { - { nameof(AccurateAllocations.EightBytesArray), 8 + objectAllocationOverhead + arraySizeOverhead }, - { nameof(AccurateAllocations.SixtyFourBytesArray), 64 + objectAllocationOverhead + arraySizeOverhead }, - - { nameof(AccurateAllocations.AllocateTask), CalculateRequiredSpace>() }, + { nameof(AccurateAllocations.SmallObjectHeap), 8 + objectAllocationOverhead + arraySizeOverhead }, }); } @@ -92,7 +88,7 @@ private void AllocateUntilGcWakesUp() } } - [Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolchain toolchain) { @@ -194,14 +190,10 @@ public byte[] SixtyFourBytesArray() } } - [Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))] - //[TheoryNetCoreOnly("Only .NET Core 2.0+ API is bug free for this case"), MemberData(nameof(GetToolchains))] + [TheoryNetCoreOnly("Only .NET Core 2.0+ API is bug free for this case"), MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { - if (toolchain is CoreRtToolchain) // the fix has not yet been backported to CoreRT - return; - long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length @@ -237,52 +229,56 @@ public void Allocate() } } - [TheoryNetCore30(".NET Core 3.0 preview6+ exposes a GC.GetTotalAllocatedBytes method which makes it possible to work"), MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain) { - if (toolchain is CoreRtToolchain) // the API has not been yet ported to CoreRT - return; - long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length long memoryAllocatedPerArray = (MultiThreadedAllocation.Size + objectAllocationOverhead + arraySizeOverhead); - long threadStartAndJoinOverhead = 112; // this is more or less a magic number taken from memory profiler - long allocatedMemoryPerThread = memoryAllocatedPerArray + threadStartAndJoinOverhead; + long maxThreadStartAndJoinOverhead = RuntimeInformation.IsFullFramework ? 2500 : 600; // these are more or less a magic numbers taken from memory profiler and test runs - AssertAllocations(toolchain, typeof(MultiThreadedAllocation), new Dictionary + AssertAllocations(toolchain, typeof(MultiThreadedAllocation), new Dictionary { - { nameof(MultiThreadedAllocation.Allocate), allocatedMemoryPerThread * MultiThreadedAllocation.ThreadsCount } + { nameof(MultiThreadedAllocation.Allocate), ( + low: memoryAllocatedPerArray * MultiThreadedAllocation.ThreadsCount, + high: (memoryAllocatedPerArray + maxThreadStartAndJoinOverhead) * MultiThreadedAllocation.ThreadsCount) } }); } private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators) { - var config = CreateConfig(toolchain); - var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config); + AssertAllocations(toolchain, benchmarkType, benchmarksAllocationsValidators, (benchmark, gcStats, expectedBytes) => + { + Assert.Equal(expectedBytes, gcStats.GetBytesAllocatedPerOperation(benchmark)); - var summary = BenchmarkRunner.Run(benchmarks); + if (expectedBytes == 0) + { + Assert.Equal(0, gcStats.GetTotalAllocatedBytes(excludeAllocationQuantumSideEffects: true)); + } + }); + } - foreach (var benchmarkAllocationsValidator in benchmarksAllocationsValidators) + private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators) + { + AssertAllocations(toolchain, benchmarkType, benchmarksAllocationsValidators, (benchmark, gcStats, expectedBytes) => { - // CoreRT is missing some of the CoreCLR threading/task related perf improvements, so sizeof(Task) calculated for CoreCLR < sizeof(Task) on CoreRT - // see https://github.com/dotnet/corert/issues/5705 for more - if (benchmarkAllocationsValidator.Key == nameof(AccurateAllocations.AllocateTask) && toolchain is CoreRtToolchain) - continue; + Assert.InRange(gcStats.GetBytesAllocatedPerOperation(benchmark), expectedBytes.low, expectedBytes.high); + }); + } - var allocatingBenchmarks = benchmarks.BenchmarksCases.Where(benchmark => benchmark.DisplayInfo.Contains(benchmarkAllocationsValidator.Key)); + private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators, Action assert) + { + var config = CreateConfig(toolchain); + var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config); - foreach (var benchmark in allocatingBenchmarks) - { - var benchmarkReport = summary.Reports.Single(report => report.BenchmarkCase == benchmark); + var summary = BenchmarkRunner.Run(benchmarks); - Assert.Equal(benchmarkAllocationsValidator.Value, benchmarkReport.GcStats.GetBytesAllocatedPerOperation(benchmark)); + foreach (var report in summary.Reports) + { + var benchmark = report.BenchmarkCase; - if (benchmarkAllocationsValidator.Value == 0) - { - Assert.Equal(0, benchmarkReport.GcStats.GetTotalAllocatedBytes(excludeAllocationQuantumSideEffects: true)); - } - } + assert(benchmark, report.GcStats, benchmarksAllocationsValidators[benchmark.Descriptor.WorkloadMethod.Name]); } } @@ -293,41 +289,9 @@ private IConfig CreateConfig(IToolchain toolchain) .WithWarmupCount(0) // don't run warmup to save some time for our CI runs .WithIterationCount(1) // single iteration is enough for us .WithGcForce(false) - .WithEnvironmentVariable("COMPlus_TieredCompilation", "0") // Tiered JIT can allocate some memory on a background thread, let's disable it to make our tests less flaky (#1542) .WithToolchain(toolchain)) .AddColumnProvider(DefaultColumnProviders.Instance) .AddDiagnoser(MemoryDiagnoser.Default) .AddLogger(toolchain.IsInProcess ? ConsoleLogger.Default : new OutputLogger(output)); // we can't use OutputLogger for the InProcess toolchains because it allocates memory on the same thread - - // note: don't copy, never use in production systems (it should work but I am not 100% sure) - private int CalculateRequiredSpace() - { - int total = SizeOfAllFields(); - - if (!typeof(T).GetTypeInfo().IsValueType) - total += IntPtr.Size * 2; // pointer to method table + object header word - - if (total % IntPtr.Size != 0) // aligning.. - total += IntPtr.Size - (total % IntPtr.Size); - - return total; - } - - // note: don't copy, never use in production systems (it should work but I am not 100% sure) - private int SizeOfAllFields() - { - int GetSize(Type type) - { - var sizeOf = typeof(Unsafe).GetTypeInfo().GetMethod(nameof(Unsafe.SizeOf)); - - return (int)sizeOf.MakeGenericMethod(type).Invoke(null, null); - } - - return typeof(T) - .GetAllFields() - .Where(field => !field.IsStatic && !field.IsLiteral) - .Distinct() - .Sum(field => GetSize(field.FieldType)); - } } } diff --git a/tests/BenchmarkDotNet.Tests/XUnit/TheoryNetCore30Attribute.cs b/tests/BenchmarkDotNet.Tests/XUnit/TheoryNetCore30Attribute.cs deleted file mode 100644 index c91de5170a..0000000000 --- a/tests/BenchmarkDotNet.Tests/XUnit/TheoryNetCore30Attribute.cs +++ /dev/null @@ -1,16 +0,0 @@ -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Portability; -using Xunit; - -namespace BenchmarkDotNet.Tests.XUnit -{ - public class TheoryNetCore30Attribute : TheoryAttribute - { - // ReSharper disable once VirtualMemberCallInConstructor - public TheoryNetCore30Attribute(string skipReason) - { - if (RuntimeInformation.GetCurrentRuntime().RuntimeMoniker != RuntimeMoniker.NetCoreApp30) - Skip = skipReason; - } - } -} \ No newline at end of file