diff --git a/src/BenchmarkDotNet/Attributes/AutomaticBaselineAttribute.cs b/src/BenchmarkDotNet/Attributes/AutomaticBaselineAttribute.cs new file mode 100644 index 0000000000..8aa0cf7cf8 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/AutomaticBaselineAttribute.cs @@ -0,0 +1,13 @@ +using BenchmarkDotNet.Configs; +using System; + +namespace BenchmarkDotNet.Attributes +{ + [AttributeUsage(AttributeTargets.Class)] + public class AutomaticBaselineAttribute : Attribute, IConfigSource + { + public IConfig Config { get; } + + public AutomaticBaselineAttribute(AutomaticBaselineMode mode) => Config = ManualConfig.CreateEmpty().WithAutomaticBaseline(mode); + } +} diff --git a/src/BenchmarkDotNet/Configs/AutomaticBaselineMode.cs b/src/BenchmarkDotNet/Configs/AutomaticBaselineMode.cs new file mode 100644 index 0000000000..5a97a78868 --- /dev/null +++ b/src/BenchmarkDotNet/Configs/AutomaticBaselineMode.cs @@ -0,0 +1,8 @@ +namespace BenchmarkDotNet.Configs +{ + public enum AutomaticBaselineMode + { + None, + Fastest + } +} diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs index 7c832b9cd3..b260e49ed0 100644 --- a/src/BenchmarkDotNet/Configs/DebugConfig.cs +++ b/src/BenchmarkDotNet/Configs/DebugConfig.cs @@ -87,5 +87,7 @@ public string ArtifactsPath public ConfigOptions Options => ConfigOptions.KeepBenchmarkFiles | ConfigOptions.DisableOptimizationsValidator; public IReadOnlyList ConfigAnalysisConclusion => emptyConclusion; + + public AutomaticBaselineMode AutomaticBaselineMode { get; } = AutomaticBaselineMode.None; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index 2ee08c8169..9e8a765aba 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -106,5 +106,7 @@ public string ArtifactsPath public IEnumerable GetFilters() => Array.Empty(); public IEnumerable GetColumnHidingRules() => Array.Empty(); + + public AutomaticBaselineMode AutomaticBaselineMode { get; } = AutomaticBaselineMode.None; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs index f3847a6f4e..36a4a1789f 100644 --- a/src/BenchmarkDotNet/Configs/IConfig.cs +++ b/src/BenchmarkDotNet/Configs/IConfig.cs @@ -57,5 +57,7 @@ public interface IConfig /// Collect any errors or warnings when composing the configuration /// IReadOnlyList ConfigAnalysisConclusion { get; } + + AutomaticBaselineMode AutomaticBaselineMode { get; } } } diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index 83b7b9374a..06be50ec7e 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -53,7 +53,8 @@ internal ImmutableConfig( SummaryStyle summaryStyle, ConfigOptions options, TimeSpan buildTimeout, - IReadOnlyList configAnalysisConclusion) + IReadOnlyList configAnalysisConclusion, + AutomaticBaselineMode automaticBaselineMode) { columnProviders = uniqueColumnProviders; loggers = uniqueLoggers; @@ -74,6 +75,7 @@ internal ImmutableConfig( Options = options; BuildTimeout = buildTimeout; ConfigAnalysisConclusion = configAnalysisConclusion; + AutomaticBaselineMode = automaticBaselineMode; } public ConfigUnionRule UnionRule { get; } @@ -83,6 +85,7 @@ internal ImmutableConfig( [NotNull] public IOrderer Orderer { get; } public SummaryStyle SummaryStyle { get; } public TimeSpan BuildTimeout { get; } + public AutomaticBaselineMode AutomaticBaselineMode { get; } public IEnumerable GetColumnProviders() => columnProviders; public IEnumerable GetExporters() => exporters; diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 677574bf86..9fdbddb668 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -71,7 +71,8 @@ public static ImmutableConfig Create(IConfig source) source.SummaryStyle ?? SummaryStyle.Default, source.Options, source.BuildTimeout, - configAnalyse.AsReadOnly() + configAnalyse.AsReadOnly(), + source.AutomaticBaselineMode ); } diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index 9f632c03d0..2897b90bd4 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -53,6 +53,7 @@ public class ManualConfig : IConfig [PublicAPI] public IOrderer Orderer { get; set; } [PublicAPI] public SummaryStyle SummaryStyle { get; set; } [PublicAPI] public TimeSpan BuildTimeout { get; set; } = DefaultConfig.Instance.BuildTimeout; + [PublicAPI] public AutomaticBaselineMode AutomaticBaselineMode { get; private set; } public IReadOnlyList ConfigAnalysisConclusion => emptyConclusion; @@ -98,6 +99,12 @@ public ManualConfig WithBuildTimeout(TimeSpan buildTimeout) return this; } + public ManualConfig WithAutomaticBaseline(AutomaticBaselineMode automaticBaselineMode) + { + AutomaticBaselineMode = automaticBaselineMode; + return this; + } + [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("This method will soon be removed, please start using .AddColumn() instead.")] public void Add(params IColumn[] newColumns) => AddColumn(newColumns); @@ -254,6 +261,10 @@ public void Add(IConfig config) columnHidingRules.AddRange(config.GetColumnHidingRules()); Options |= config.Options; BuildTimeout = GetBuildTimeout(BuildTimeout, config.BuildTimeout); + if (config.AutomaticBaselineMode != AutomaticBaselineMode.None) + { + AutomaticBaselineMode = config.AutomaticBaselineMode; + } } /// diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 08a1efecc9..aa225e5c05 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -37,6 +37,7 @@ public class Summary private ImmutableDictionary ReportMap { get; } private BaseliningStrategy BaseliningStrategy { get; } private bool? isMultipleRuntimes; + private readonly BenchmarkCase inferredBaselineBenchmarkCase; public Summary( string title, @@ -62,6 +63,7 @@ public Summary( DisplayPrecisionManager = new DisplayPrecisionManager(this); Orderer = GetConfiguredOrdererOrDefaultOne(reports.Select(report => report.BenchmarkCase.Config)); BenchmarksCases = Orderer.GetSummaryOrder(reports.Select(report => report.BenchmarkCase).ToImmutableArray(), this).ToImmutableArray(); // we sort it first + inferredBaselineBenchmarkCase = GetFastestBenchmarkCase(reports); Reports = BenchmarksCases.Select(b => ReportMap[b]).ToImmutableArray(); // we use sorted collection to re-create reports list BaseliningStrategy = BaseliningStrategy.Create(BenchmarksCases); Style = GetConfiguredSummaryStyleOrDefaultOne(BenchmarksCases).WithCultureInfo(cultureInfo); @@ -69,6 +71,27 @@ public Summary( AllRuntimes = BuildAllRuntimes(HostEnvironmentInfo, Reports); } + private static BenchmarkCase GetFastestBenchmarkCase(ImmutableArray reports) + { + if (reports.Any(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest)) + { + var fastestReport = reports.First(); + if (fastestReport.ResultStatistics != null) + { + foreach (var report in reports.Skip(1).Where(r => r.ResultStatistics != null)) + { + if (report.ResultStatistics.Mean < fastestReport.ResultStatistics.Mean) + { + fastestReport = report; + } + } + } + return fastestReport.BenchmarkCase; + } + + return null; + } + [PublicAPI] public bool HasReport(BenchmarkCase benchmarkCase) => ReportMap.ContainsKey(benchmarkCase); /// @@ -133,7 +156,11 @@ public string GetLogicalGroupKey(BenchmarkCase benchmarkCase) => Orderer.GetLogicalGroupKey(BenchmarksCases, benchmarkCase); public bool IsBaseline(BenchmarkCase benchmarkCase) - => BaseliningStrategy.IsBaseline(benchmarkCase); + { + return inferredBaselineBenchmarkCase != null + ? inferredBaselineBenchmarkCase == benchmarkCase + : BaseliningStrategy.IsBaseline(benchmarkCase); + } [CanBeNull] public BenchmarkCase GetBaseline(string logicalGroupKey) diff --git a/src/BenchmarkDotNet/Validators/BaselineValidator.cs b/src/BenchmarkDotNet/Validators/BaselineValidator.cs index 085710e3a5..12406fdcf6 100644 --- a/src/BenchmarkDotNet/Validators/BaselineValidator.cs +++ b/src/BenchmarkDotNet/Validators/BaselineValidator.cs @@ -14,6 +14,17 @@ private BaselineValidator() { } public IEnumerable Validate(ValidationParameters input) { + if (input.Config.AutomaticBaselineMode != Configs.AutomaticBaselineMode.None) + { + foreach (var benchmark in input.Benchmarks) + { + if (benchmark.Descriptor.Baseline || benchmark.Job.Meta.Baseline) + { + yield return new ValidationError(TreatsWarningsAsErrors, "You cannot use both pre-configured and automatic baseline configuration", benchmark); + } + } + } + var allBenchmarks = input.Benchmarks.ToImmutableArray(); var orderProvider = input.Config.Orderer; diff --git a/tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs new file mode 100644 index 0000000000..357fd96d79 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs @@ -0,0 +1,32 @@ +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Tests.Mocks; +using System; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class AutomaticBaselineTests + { + [Fact] + public void AutomaticBaselineSelectionIsCorrect() + { + var config = ManualConfig.CreateEmpty() + .AddColumnProvider(DefaultColumnProviders.Instance) + .WithAutomaticBaseline(AutomaticBaselineMode.Fastest); + + var summary = MockFactory.CreateSummary(config, hugeSd: true, Array.Empty()); + var table = summary.GetTable(SummaryStyle.Default); + var method = table.Columns.Single(c => c.Header == "Method"); + var ratio = table.Columns.Single(c => c.Header == "Ratio"); + + Assert.Equal(2, method.Content.Length); + Assert.Equal(nameof(MockFactory.MockBenchmarkClass.Foo), method.Content[0]); + Assert.Equal(1.0, double.Parse(ratio.Content[0])); // A faster one, see measurements in MockFactory.cs + Assert.Equal(nameof(MockFactory.MockBenchmarkClass.Bar), method.Content[1]); + Assert.Equal(1.5, double.Parse(ratio.Content[1])); + } + } +}