-
-
Notifications
You must be signed in to change notification settings - Fork 1k
Add expected results tests #2361
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit
Hold shift + click to select a range
8543aec
Added manual tests for expected results.
timcassell 6dfe6f4
Use a fallback value for cpu resolution.
timcassell e0959b9
Add multiple struct sizes.
timcassell 78ce31d
Use same code as ZeroMeasurementAnalyser.
timcassell 7f163c8
Use ZeroMeasurementHelper for both checks.
timcassell a6a4625
Set benchmark process priority to realtime.
timcassell bb41ff3
Add threshold argument to ZeroMeasurementHelper.
timcassell 3919a37
Friendly benchmark names
timcassell 1dd6c84
Removed RealTimeBenchmarks
timcassell File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
220 changes: 220 additions & 0 deletions
220
tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,220 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using BenchmarkDotNet.Attributes; | ||
| using BenchmarkDotNet.Configs; | ||
| using BenchmarkDotNet.Diagnosers; | ||
| using BenchmarkDotNet.Extensions; | ||
| using BenchmarkDotNet.Jobs; | ||
| using BenchmarkDotNet.Portability; | ||
| using BenchmarkDotNet.Reports; | ||
| using BenchmarkDotNet.Tests.XUnit; | ||
| using BenchmarkDotNet.Toolchains.InProcess.Emit; | ||
| using Xunit; | ||
| using Xunit.Abstractions; | ||
|
|
||
| namespace BenchmarkDotNet.IntegrationTests.ManualRunning | ||
| { | ||
| public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor | ||
| { | ||
| // NativeAot takes a long time to build, so not including it in these tests. | ||
| // We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains. | ||
|
|
||
| public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { } | ||
|
|
||
| private static IEnumerable<Type> EmptyBenchmarkTypes() => | ||
| new[] | ||
| { | ||
| typeof(EmptyVoid), | ||
| typeof(EmptyByte), | ||
| typeof(EmptySByte), | ||
| typeof(EmptyShort), | ||
| typeof(EmptyUShort), | ||
| typeof(EmptyChar), | ||
| typeof(EmptyInt32), | ||
| typeof(EmptyUInt32), | ||
| typeof(EmptyInt64), | ||
| typeof(EmptyUInt64), | ||
| typeof(EmptyIntPtr), | ||
| typeof(EmptyUIntPtr), | ||
| typeof(EmptyVoidPointer), | ||
| typeof(EmptyClass) | ||
| }; | ||
|
|
||
| public static IEnumerable<object[]> InProcessData() | ||
| { | ||
| foreach (var type in EmptyBenchmarkTypes()) | ||
| { | ||
| yield return new object[] { type }; | ||
| } | ||
| } | ||
|
|
||
| public static IEnumerable<object[]> CoreData() | ||
| { | ||
| foreach (var type in EmptyBenchmarkTypes()) | ||
| { | ||
| yield return new object[] { type, RuntimeMoniker.Net70 }; | ||
| yield return new object[] { type, RuntimeMoniker.Mono70 }; | ||
| } | ||
| } | ||
|
|
||
| public static IEnumerable<object[]> FrameworkData() | ||
| { | ||
| foreach (var type in EmptyBenchmarkTypes()) | ||
| { | ||
| yield return new object[] { type, RuntimeMoniker.Net462 }; | ||
| yield return new object[] { type, RuntimeMoniker.Mono }; | ||
| } | ||
| } | ||
|
|
||
| [Theory] | ||
| [MemberData(nameof(InProcessData))] | ||
| public void EmptyBenchmarksReportZeroTimeAndAllocated_InProcess(Type benchmarkType) | ||
| { | ||
| AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() | ||
| .AddJob(Job.Default | ||
| .WithToolchain(InProcessEmitToolchain.Instance) | ||
| )); | ||
| } | ||
|
|
||
| [TheoryNetCoreOnly("To not repeat tests in both full Framework and Core")] | ||
| [MemberData(nameof(CoreData))] | ||
| public void EmptyBenchmarksReportZeroTimeAndAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker) | ||
| { | ||
| AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() | ||
| .AddJob(Job.Default | ||
| .WithRuntime(runtimeMoniker.GetRuntime()) | ||
| )); | ||
| } | ||
|
|
||
| [TheoryFullFrameworkOnly("Can only run full Framework and Mono tests from Framework host")] | ||
| [MemberData(nameof(FrameworkData))] | ||
| public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker) | ||
| { | ||
| AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty() | ||
| .AddJob(Job.Default | ||
| .WithRuntime(runtimeMoniker.GetRuntime()) | ||
| )); | ||
| } | ||
|
|
||
| private void AssertZeroResults(Type benchmarkType, IConfig config) | ||
| { | ||
| var summary = CanExecute(benchmarkType, config | ||
| .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond)) | ||
| .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) | ||
| ); | ||
|
|
||
| var cpuGhz = RuntimeInformation.GetCpuInfo().MaxFrequency.Value.ToGHz(); | ||
|
|
||
| foreach (var report in summary.Reports) | ||
| { | ||
| Assert.True(cpuGhz * report.ResultStatistics.Mean < 1, $"Actual time was greater than 1 clock cycle."); | ||
|
|
||
| var overheadTime = report.AllMeasurements | ||
| .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) | ||
| .Select(m => m.GetAverageTime().Nanoseconds) | ||
| .Average(); | ||
|
|
||
| var workloadTime = report.AllMeasurements | ||
| .Where(m => m.IsWorkload() && m.IterationStage == Engines.IterationStage.Actual) | ||
| .Select(m => m.GetAverageTime().Nanoseconds) | ||
| .Average(); | ||
|
|
||
| // Allow for 1 cpu cycle variance | ||
| Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload."); | ||
|
|
||
| Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); | ||
| } | ||
| } | ||
|
|
||
| [Fact] | ||
| public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess() | ||
| { | ||
| AssertLargeStructResults(ManualConfig.CreateEmpty() | ||
| .AddJob(Job.Default | ||
| .WithToolchain(InProcessEmitToolchain.Instance) | ||
| )); | ||
| } | ||
|
|
||
| [TheoryNetCoreOnly("To not repeat tests in both full Framework and Core")] | ||
| [InlineData(RuntimeMoniker.Net70)] | ||
| [InlineData(RuntimeMoniker.Mono70)] | ||
| public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(RuntimeMoniker runtimeMoniker) | ||
| { | ||
| AssertLargeStructResults(ManualConfig.CreateEmpty() | ||
| .AddJob(Job.Default | ||
| .WithRuntime(runtimeMoniker.GetRuntime()) | ||
| )); | ||
| } | ||
|
|
||
| [TheoryFullFrameworkOnly("Can only run full Framework and Mono tests from Framework host")] | ||
| [InlineData(RuntimeMoniker.Net462)] | ||
| [InlineData(RuntimeMoniker.Mono)] | ||
| public void LargeStructBenchmarksReportsNonZeroTimeAndZeroAllocated_Framework(RuntimeMoniker runtimeMoniker) | ||
| { | ||
| AssertLargeStructResults(ManualConfig.CreateEmpty() | ||
| .AddJob(Job.Default | ||
| .WithRuntime(runtimeMoniker.GetRuntime()) | ||
| )); | ||
| } | ||
|
|
||
| private void AssertLargeStructResults(IConfig config) | ||
| { | ||
| var summary = CanExecute<LargeStruct>(config | ||
| .WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(Perfolizer.Horology.TimeUnit.Nanosecond)) | ||
| .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) | ||
| ); | ||
|
|
||
| var cpuGhz = RuntimeInformation.GetCpuInfo().MaxFrequency.Value.ToGHz(); | ||
|
|
||
| foreach (var report in summary.Reports) | ||
| { | ||
| Assert.True(cpuGhz * report.ResultStatistics.Mean >= 1, $"Actual time was less than 1 clock cycle."); | ||
timcassell marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| var overheadTime = report.AllMeasurements | ||
| .Where(m => m.IsOverhead() && m.IterationStage == Engines.IterationStage.Actual) | ||
| .Select(m => m.GetAverageTime().Nanoseconds) | ||
| .Average(); | ||
|
|
||
| var workloadTime = report.AllMeasurements | ||
| .Where(m => m.IsWorkload() && m.IterationStage == Engines.IterationStage.Actual) | ||
| .Select(m => m.GetAverageTime().Nanoseconds) | ||
| .Average(); | ||
|
|
||
| // Allow for 1 cpu cycle variance | ||
| Assert.True(overheadTime * cpuGhz < workloadTime * cpuGhz + 1, "Overhead took more time than workload."); | ||
|
|
||
| Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public class LargeStruct | ||
timcassell marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| { | ||
| public struct Struct | ||
| { | ||
| // 128 bits | ||
| public long l1, l2, l3, l4, | ||
| l5, l6, l7, l8, | ||
| l9, l10, l11, l12, | ||
| l13, l14, l15, l16; | ||
| } | ||
|
|
||
| [Benchmark] public Struct Benchmark() => default; | ||
| } | ||
| } | ||
|
|
||
| public class EmptyVoid { [Benchmark] public void Benchmark() { } } | ||
| public class EmptyByte { [Benchmark] public byte Benchmark() => default; } | ||
| public class EmptySByte { [Benchmark] public sbyte Benchmark() => default; } | ||
| public class EmptyShort { [Benchmark] public short Benchmark() => default; } | ||
| public class EmptyUShort { [Benchmark] public ushort Benchmark() => default; } | ||
| public class EmptyChar { [Benchmark] public char Benchmark() => default; } | ||
| public class EmptyInt32 { [Benchmark] public int Benchmark() => default; } | ||
| public class EmptyUInt32 { [Benchmark] public uint Benchmark() => default; } | ||
| public class EmptyInt64 { [Benchmark] public long Benchmark() => default; } | ||
| public class EmptyUInt64 { [Benchmark] public ulong Benchmark() => default; } | ||
| public class EmptyIntPtr { [Benchmark] public IntPtr Benchmark() => default; } | ||
| public class EmptyUIntPtr { [Benchmark] public UIntPtr Benchmark() => default; } | ||
| public class EmptyVoidPointer { [Benchmark] public unsafe void* Benchmark() => default; } | ||
| public class EmptyClass { [Benchmark] public object Class() => default; } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.