From deedc64b0a40d4878a1f1b56a9eabb0713e6b77a Mon Sep 17 00:00:00 2001 From: Serg046 Date: Wed, 26 Oct 2022 19:11:29 +0300 Subject: [PATCH 1/6] Issue #1024: Calculate baseline by the fastest benchmark --- .../Attributes/AutomaticBaselineAttribute.cs | 13 +++++ .../Configs/AutomaticBaselineMode.cs | 8 +++ src/BenchmarkDotNet/Configs/DebugConfig.cs | 2 + src/BenchmarkDotNet/Configs/DefaultConfig.cs | 2 + src/BenchmarkDotNet/Configs/IConfig.cs | 2 + .../Configs/ImmutableConfig.cs | 5 +- .../Configs/ImmutableConfigBuilder.cs | 3 +- src/BenchmarkDotNet/Configs/ManualConfig.cs | 8 +++ src/BenchmarkDotNet/Reports/Summary.cs | 26 ++++++++- .../AutomaticBaselineTests.cs | 53 +++++++++++++++++++ 10 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 src/BenchmarkDotNet/Attributes/AutomaticBaselineAttribute.cs create mode 100644 src/BenchmarkDotNet/Configs/AutomaticBaselineMode.cs create mode 100644 tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs diff --git a/src/BenchmarkDotNet/Attributes/AutomaticBaselineAttribute.cs b/src/BenchmarkDotNet/Attributes/AutomaticBaselineAttribute.cs new file mode 100644 index 0000000000..8aa0cf7cf8 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/AutomaticBaselineAttribute.cs @@ -0,0 +1,13 @@ +using BenchmarkDotNet.Configs; +using System; + +namespace BenchmarkDotNet.Attributes +{ + [AttributeUsage(AttributeTargets.Class)] + public class AutomaticBaselineAttribute : Attribute, IConfigSource + { + public IConfig Config { get; } + + public AutomaticBaselineAttribute(AutomaticBaselineMode mode) => Config = ManualConfig.CreateEmpty().WithAutomaticBaseline(mode); + } +} diff --git a/src/BenchmarkDotNet/Configs/AutomaticBaselineMode.cs b/src/BenchmarkDotNet/Configs/AutomaticBaselineMode.cs new file mode 100644 index 0000000000..5a97a78868 --- /dev/null +++ b/src/BenchmarkDotNet/Configs/AutomaticBaselineMode.cs @@ -0,0 +1,8 @@ +namespace BenchmarkDotNet.Configs +{ + public enum AutomaticBaselineMode + { + None, + Fastest + } +} diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs index 7c832b9cd3..b260e49ed0 100644 --- a/src/BenchmarkDotNet/Configs/DebugConfig.cs +++ b/src/BenchmarkDotNet/Configs/DebugConfig.cs @@ -87,5 +87,7 @@ public string ArtifactsPath public ConfigOptions Options => ConfigOptions.KeepBenchmarkFiles | ConfigOptions.DisableOptimizationsValidator; public IReadOnlyList ConfigAnalysisConclusion => emptyConclusion; + + public AutomaticBaselineMode AutomaticBaselineMode { get; } = AutomaticBaselineMode.None; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index 2ee08c8169..9e8a765aba 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -106,5 +106,7 @@ public string ArtifactsPath public IEnumerable GetFilters() => Array.Empty(); public IEnumerable GetColumnHidingRules() => Array.Empty(); + + public AutomaticBaselineMode AutomaticBaselineMode { get; } = AutomaticBaselineMode.None; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs index f3847a6f4e..36a4a1789f 100644 --- a/src/BenchmarkDotNet/Configs/IConfig.cs +++ b/src/BenchmarkDotNet/Configs/IConfig.cs @@ -57,5 +57,7 @@ public interface IConfig /// Collect any errors or warnings when composing the configuration /// IReadOnlyList ConfigAnalysisConclusion { get; } + + AutomaticBaselineMode AutomaticBaselineMode { get; } } } diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index 83b7b9374a..06be50ec7e 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -53,7 +53,8 @@ internal ImmutableConfig( SummaryStyle summaryStyle, ConfigOptions options, TimeSpan buildTimeout, - IReadOnlyList configAnalysisConclusion) + IReadOnlyList configAnalysisConclusion, + AutomaticBaselineMode automaticBaselineMode) { columnProviders = uniqueColumnProviders; loggers = uniqueLoggers; @@ -74,6 +75,7 @@ internal ImmutableConfig( Options = options; BuildTimeout = buildTimeout; ConfigAnalysisConclusion = configAnalysisConclusion; + AutomaticBaselineMode = automaticBaselineMode; } public ConfigUnionRule UnionRule { get; } @@ -83,6 +85,7 @@ internal ImmutableConfig( [NotNull] public IOrderer Orderer { get; } public SummaryStyle SummaryStyle { get; } public TimeSpan BuildTimeout { get; } + public AutomaticBaselineMode AutomaticBaselineMode { get; } public IEnumerable GetColumnProviders() => columnProviders; public IEnumerable GetExporters() => exporters; diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 677574bf86..9fdbddb668 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -71,7 +71,8 @@ public static ImmutableConfig Create(IConfig source) source.SummaryStyle ?? SummaryStyle.Default, source.Options, source.BuildTimeout, - configAnalyse.AsReadOnly() + configAnalyse.AsReadOnly(), + source.AutomaticBaselineMode ); } diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index 9f632c03d0..c6248d6de0 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -53,6 +53,7 @@ public class ManualConfig : IConfig [PublicAPI] public IOrderer Orderer { get; set; } [PublicAPI] public SummaryStyle SummaryStyle { get; set; } [PublicAPI] public TimeSpan BuildTimeout { get; set; } = DefaultConfig.Instance.BuildTimeout; + [PublicAPI] public AutomaticBaselineMode AutomaticBaselineMode { get; private set; } public IReadOnlyList ConfigAnalysisConclusion => emptyConclusion; @@ -98,6 +99,12 @@ public ManualConfig WithBuildTimeout(TimeSpan buildTimeout) return this; } + public ManualConfig WithAutomaticBaseline(AutomaticBaselineMode automaticBaselineMode) + { + AutomaticBaselineMode = automaticBaselineMode; + return this; + } + [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("This method will soon be removed, please start using .AddColumn() instead.")] public void Add(params IColumn[] newColumns) => AddColumn(newColumns); @@ -254,6 +261,7 @@ public void Add(IConfig config) columnHidingRules.AddRange(config.GetColumnHidingRules()); Options |= config.Options; BuildTimeout = GetBuildTimeout(BuildTimeout, config.BuildTimeout); + AutomaticBaselineMode = config.AutomaticBaselineMode; } /// diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 08a1efecc9..e560a89de0 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -37,6 +37,7 @@ public class Summary private ImmutableDictionary ReportMap { get; } private BaseliningStrategy BaseliningStrategy { get; } private bool? isMultipleRuntimes; + private readonly BenchmarkCase inferredBaselineBenchmarkCase; public Summary( string title, @@ -62,6 +63,7 @@ public Summary( DisplayPrecisionManager = new DisplayPrecisionManager(this); Orderer = GetConfiguredOrdererOrDefaultOne(reports.Select(report => report.BenchmarkCase.Config)); BenchmarksCases = Orderer.GetSummaryOrder(reports.Select(report => report.BenchmarkCase).ToImmutableArray(), this).ToImmutableArray(); // we sort it first + inferredBaselineBenchmarkCase = GetFastestBenchmarkCase(reports); Reports = BenchmarksCases.Select(b => ReportMap[b]).ToImmutableArray(); // we use sorted collection to re-create reports list BaseliningStrategy = BaseliningStrategy.Create(BenchmarksCases); Style = GetConfiguredSummaryStyleOrDefaultOne(BenchmarksCases).WithCultureInfo(cultureInfo); @@ -69,6 +71,24 @@ public Summary( AllRuntimes = BuildAllRuntimes(HostEnvironmentInfo, Reports); } + private static BenchmarkCase GetFastestBenchmarkCase(ImmutableArray reports) + { + if (reports.Any() && reports.All(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest)) + { + var fastestReport = reports.First(); + foreach (var report in reports.Skip(1)) + { + if (report.ResultStatistics.Mean < fastestReport.ResultStatistics.Mean) + { + fastestReport = report; + } + } + return fastestReport.BenchmarkCase; + } + + return null; + } + [PublicAPI] public bool HasReport(BenchmarkCase benchmarkCase) => ReportMap.ContainsKey(benchmarkCase); /// @@ -133,7 +153,11 @@ public string GetLogicalGroupKey(BenchmarkCase benchmarkCase) => Orderer.GetLogicalGroupKey(BenchmarksCases, benchmarkCase); public bool IsBaseline(BenchmarkCase benchmarkCase) - => BaseliningStrategy.IsBaseline(benchmarkCase); + { + return inferredBaselineBenchmarkCase != null + ? inferredBaselineBenchmarkCase == benchmarkCase + : BaseliningStrategy.IsBaseline(benchmarkCase); + } [CanBeNull] public BenchmarkCase GetBaseline(string logicalGroupKey) diff --git a/tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs new file mode 100644 index 0000000000..acd5cbc7d8 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs @@ -0,0 +1,53 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using System.Linq; +using System.Threading; +using Xunit; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class AutomaticBaselineTests + { + [Fact] + public void AutomaticBaselineSelectionIsCorrect() + { + var config = CreateConfig(); + + var summary = BenchmarkRunner.Run(); + + var table = summary.GetTable(SummaryStyle.Default); + var column = table.Columns.Single(c => c.Header == "Ratio"); + Assert.Equal(2, column.Content.Length); + Assert.Equal(1.0, double.Parse(column.Content[1])); // Ratio of TwoMilliseconds + Assert.True(double.Parse(column.Content[0]) > 1.0); // Ratio of TwoHundredMilliseconds + } + + [AutomaticBaseline(AutomaticBaselineMode.Fastest)] + public class BaselineSample + { + [Benchmark] + public void TwoHundredMilliseconds() + { + Thread.Sleep(200); + } + + [Benchmark] + public void TwoMilliseconds() + { + Thread.Sleep(2); + } + } + + private IConfig CreateConfig() + => ManualConfig.CreateEmpty() + .AddJob(Job.ShortRun + .WithEvaluateOverhead(false) // no need to run idle for this test + .WithWarmupCount(0) // don't run warmup to save some time for our CI runs + .WithIterationCount(1)) // single iteration is enough for us + .AddColumnProvider(DefaultColumnProviders.Instance); + } +} From fd9f34fc837d3bebfeca888506e4c6a5fcbec232 Mon Sep 17 00:00:00 2001 From: Serg046 Date: Wed, 26 Oct 2022 23:25:03 +0300 Subject: [PATCH 2/6] Issue #1024: Add validations --- src/BenchmarkDotNet/Reports/Summary.cs | 9 ++++++--- src/BenchmarkDotNet/Validators/BaselineValidator.cs | 11 +++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index e560a89de0..3d260c3d12 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -76,11 +76,14 @@ private static BenchmarkCase GetFastestBenchmarkCase(ImmutableArray r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest)) { var fastestReport = reports.First(); - foreach (var report in reports.Skip(1)) + if (fastestReport.ResultStatistics != null) { - if (report.ResultStatistics.Mean < fastestReport.ResultStatistics.Mean) + foreach (var report in reports.Skip(1).Where(r => r.ResultStatistics != null)) { - fastestReport = report; + if (report.ResultStatistics.Mean < fastestReport.ResultStatistics.Mean) + { + fastestReport = report; + } } } return fastestReport.BenchmarkCase; diff --git a/src/BenchmarkDotNet/Validators/BaselineValidator.cs b/src/BenchmarkDotNet/Validators/BaselineValidator.cs index 085710e3a5..12406fdcf6 100644 --- a/src/BenchmarkDotNet/Validators/BaselineValidator.cs +++ b/src/BenchmarkDotNet/Validators/BaselineValidator.cs @@ -14,6 +14,17 @@ private BaselineValidator() { } public IEnumerable Validate(ValidationParameters input) { + if (input.Config.AutomaticBaselineMode != Configs.AutomaticBaselineMode.None) + { + foreach (var benchmark in input.Benchmarks) + { + if (benchmark.Descriptor.Baseline || benchmark.Job.Meta.Baseline) + { + yield return new ValidationError(TreatsWarningsAsErrors, "You cannot use both pre-configured and automatic baseline configuration", benchmark); + } + } + } + var allBenchmarks = input.Benchmarks.ToImmutableArray(); var orderProvider = input.Config.Orderer; From d20c117b567f0ddc9effd56eeda9c299d97b8ece Mon Sep 17 00:00:00 2001 From: Serg046 Date: Mon, 31 Oct 2022 02:24:23 +0300 Subject: [PATCH 3/6] Issue #1024: Correct configuration --- src/BenchmarkDotNet/Configs/ManualConfig.cs | 5 ++++- src/BenchmarkDotNet/Reports/Summary.cs | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index c6248d6de0..2897b90bd4 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -261,7 +261,10 @@ public void Add(IConfig config) columnHidingRules.AddRange(config.GetColumnHidingRules()); Options |= config.Options; BuildTimeout = GetBuildTimeout(BuildTimeout, config.BuildTimeout); - AutomaticBaselineMode = config.AutomaticBaselineMode; + if (config.AutomaticBaselineMode != AutomaticBaselineMode.None) + { + AutomaticBaselineMode = config.AutomaticBaselineMode; + } } /// diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 3d260c3d12..ad45eedbc9 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -73,7 +73,8 @@ public Summary( private static BenchmarkCase GetFastestBenchmarkCase(ImmutableArray reports) { - if (reports.Any() && reports.All(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest)) + var baselineReport = reports.Where(r => r.BenchmarkCase.Config.AutomaticBaselineMode != AutomaticBaselineMode.None).LastOrDefault(); + if (baselineReport != null && baselineReport.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest) { var fastestReport = reports.First(); if (fastestReport.ResultStatistics != null) From d8d99e2a42cb82ae299f4be60ac3c073671665c4 Mon Sep 17 00:00:00 2001 From: Serg046 Date: Mon, 31 Oct 2022 03:08:16 +0300 Subject: [PATCH 4/6] Issue #1024: Make AutomaticBaselineSelectionIsCorrect test stable enough --- .../AutomaticBaselineTests.cs | 49 ++++++------------- 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs index acd5cbc7d8..357fd96d79 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs @@ -1,11 +1,9 @@ -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Mocks; +using System; using System.Linq; -using System.Threading; using Xunit; namespace BenchmarkDotNet.IntegrationTests @@ -15,39 +13,20 @@ public class AutomaticBaselineTests [Fact] public void AutomaticBaselineSelectionIsCorrect() { - var config = CreateConfig(); - - var summary = BenchmarkRunner.Run(); + var config = ManualConfig.CreateEmpty() + .AddColumnProvider(DefaultColumnProviders.Instance) + .WithAutomaticBaseline(AutomaticBaselineMode.Fastest); + var summary = MockFactory.CreateSummary(config, hugeSd: true, Array.Empty()); var table = summary.GetTable(SummaryStyle.Default); - var column = table.Columns.Single(c => c.Header == "Ratio"); - Assert.Equal(2, column.Content.Length); - Assert.Equal(1.0, double.Parse(column.Content[1])); // Ratio of TwoMilliseconds - Assert.True(double.Parse(column.Content[0]) > 1.0); // Ratio of TwoHundredMilliseconds - } + var method = table.Columns.Single(c => c.Header == "Method"); + var ratio = table.Columns.Single(c => c.Header == "Ratio"); - [AutomaticBaseline(AutomaticBaselineMode.Fastest)] - public class BaselineSample - { - [Benchmark] - public void TwoHundredMilliseconds() - { - Thread.Sleep(200); - } - - [Benchmark] - public void TwoMilliseconds() - { - Thread.Sleep(2); - } + Assert.Equal(2, method.Content.Length); + Assert.Equal(nameof(MockFactory.MockBenchmarkClass.Foo), method.Content[0]); + Assert.Equal(1.0, double.Parse(ratio.Content[0])); // A faster one, see measurements in MockFactory.cs + Assert.Equal(nameof(MockFactory.MockBenchmarkClass.Bar), method.Content[1]); + Assert.Equal(1.5, double.Parse(ratio.Content[1])); } - - private IConfig CreateConfig() - => ManualConfig.CreateEmpty() - .AddJob(Job.ShortRun - .WithEvaluateOverhead(false) // no need to run idle for this test - .WithWarmupCount(0) // don't run warmup to save some time for our CI runs - .WithIterationCount(1)) // single iteration is enough for us - .AddColumnProvider(DefaultColumnProviders.Instance); } } From 873243d13515c548f74303d6775f2171fb162288 Mon Sep 17 00:00:00 2001 From: Sergey Aseev Date: Mon, 31 Oct 2022 16:08:27 +0300 Subject: [PATCH 5/6] Update src/BenchmarkDotNet/Reports/Summary.cs Co-authored-by: Yegor Stepanov --- src/BenchmarkDotNet/Reports/Summary.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index ad45eedbc9..6fb63b7908 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -73,8 +73,7 @@ public Summary( private static BenchmarkCase GetFastestBenchmarkCase(ImmutableArray reports) { - var baselineReport = reports.Where(r => r.BenchmarkCase.Config.AutomaticBaselineMode != AutomaticBaselineMode.None).LastOrDefault(); - if (baselineReport != null && baselineReport.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest) + if (reports.Any(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest) { var fastestReport = reports.First(); if (fastestReport.ResultStatistics != null) From 885bc049cef1b7d1012faddd4d90ce2c9380bdb9 Mon Sep 17 00:00:00 2001 From: Serg046 Date: Mon, 31 Oct 2022 16:12:06 +0300 Subject: [PATCH 6/6] Fix a typo --- src/BenchmarkDotNet/Reports/Summary.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 6fb63b7908..aa225e5c05 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -73,7 +73,7 @@ public Summary( private static BenchmarkCase GetFastestBenchmarkCase(ImmutableArray reports) { - if (reports.Any(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest) + if (reports.Any(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest)) { var fastestReport = reports.First(); if (fastestReport.ResultStatistics != null)