Issue #1024: Calculate baseline by the fastest benchmark

src/BenchmarkDotNet/Attributes/AutomaticBaselineAttribute.cs

@@ -0,0 +1,13 @@
+using BenchmarkDotNet.Configs;
+using System;
+
+namespace BenchmarkDotNet.Attributes
+{
+    [AttributeUsage(AttributeTargets.Class)]
+    public class AutomaticBaselineAttribute : Attribute, IConfigSource
+    {
+        public IConfig Config { get; }
+
+        public AutomaticBaselineAttribute(AutomaticBaselineMode mode) => Config = ManualConfig.CreateEmpty().WithAutomaticBaseline(mode);
+    }
+}

src/BenchmarkDotNet/Configs/AutomaticBaselineMode.cs

@@ -0,0 +1,8 @@
+namespace BenchmarkDotNet.Configs
+{
+    public enum AutomaticBaselineMode
+    {
+        None,
+        Fastest
+    }
+}

src/BenchmarkDotNet/Configs/DebugConfig.cs

@@ -87,5 +87,7 @@ public string ArtifactsPath
         public ConfigOptions Options => ConfigOptions.KeepBenchmarkFiles | ConfigOptions.DisableOptimizationsValidator;
 
         public IReadOnlyList<Conclusion> ConfigAnalysisConclusion => emptyConclusion;
+
+        public AutomaticBaselineMode AutomaticBaselineMode { get; } = AutomaticBaselineMode.None;
     }
 }

src/BenchmarkDotNet/Configs/DefaultConfig.cs

@@ -106,5 +106,7 @@ public string ArtifactsPath
         public IEnumerable<IFilter> GetFilters() => Array.Empty<IFilter>();
 
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => Array.Empty<IColumnHidingRule>();
+
+        public AutomaticBaselineMode AutomaticBaselineMode { get; } = AutomaticBaselineMode.None;
     }
 }

src/BenchmarkDotNet/Configs/IConfig.cs

@@ -57,5 +57,7 @@ public interface IConfig
         /// Collect any errors or warnings when composing the configuration
         /// </summary>
         IReadOnlyList<Conclusion> ConfigAnalysisConclusion { get; }
+
+        AutomaticBaselineMode AutomaticBaselineMode { get; }
     }
 }

src/BenchmarkDotNet/Configs/ImmutableConfig.cs

@@ -53,7 +53,8 @@ internal ImmutableConfig(
             SummaryStyle summaryStyle,
             ConfigOptions options,
             TimeSpan buildTimeout,
-            IReadOnlyList<Conclusion> configAnalysisConclusion)
+            IReadOnlyList<Conclusion> configAnalysisConclusion,
+            AutomaticBaselineMode automaticBaselineMode)
         {
             columnProviders = uniqueColumnProviders;
             loggers = uniqueLoggers;
@@ -74,6 +75,7 @@ internal ImmutableConfig(
             Options = options;
             BuildTimeout = buildTimeout;
             ConfigAnalysisConclusion = configAnalysisConclusion;
+            AutomaticBaselineMode = automaticBaselineMode;
         }
 
         public ConfigUnionRule UnionRule { get; }
@@ -83,6 +85,7 @@ internal ImmutableConfig(
         [NotNull] public IOrderer Orderer { get; }
         public SummaryStyle SummaryStyle { get; }
         public TimeSpan BuildTimeout { get; }
+        public AutomaticBaselineMode AutomaticBaselineMode { get; }
 
         public IEnumerable<IColumnProvider> GetColumnProviders() => columnProviders;
         public IEnumerable<IExporter> GetExporters() => exporters;

src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs

@@ -71,7 +71,8 @@ public static ImmutableConfig Create(IConfig source)
                 source.SummaryStyle ?? SummaryStyle.Default,
                 source.Options,
                 source.BuildTimeout,
-                configAnalyse.AsReadOnly()
+                configAnalyse.AsReadOnly(),
+                source.AutomaticBaselineMode
             );
         }
 

src/BenchmarkDotNet/Configs/ManualConfig.cs

@@ -53,6 +53,7 @@ public class ManualConfig : IConfig
         [PublicAPI] public IOrderer Orderer { get; set; }
         [PublicAPI] public SummaryStyle SummaryStyle { get; set; }
         [PublicAPI] public TimeSpan BuildTimeout { get; set; } = DefaultConfig.Instance.BuildTimeout;
+        [PublicAPI] public AutomaticBaselineMode AutomaticBaselineMode { get; private set; }
 
         public IReadOnlyList<Conclusion> ConfigAnalysisConclusion => emptyConclusion;
 
@@ -98,6 +99,12 @@ public ManualConfig WithBuildTimeout(TimeSpan buildTimeout)
             return this;
         }
 
+        public ManualConfig WithAutomaticBaseline(AutomaticBaselineMode automaticBaselineMode)
+        {
+            AutomaticBaselineMode = automaticBaselineMode;
+            return this;
+        }
+
         [EditorBrowsable(EditorBrowsableState.Never)]
         [Obsolete("This method will soon be removed, please start using .AddColumn() instead.")]
         public void Add(params IColumn[] newColumns) => AddColumn(newColumns);
@@ -254,6 +261,10 @@ public void Add(IConfig config)
             columnHidingRules.AddRange(config.GetColumnHidingRules());
             Options |= config.Options;
             BuildTimeout = GetBuildTimeout(BuildTimeout, config.BuildTimeout);
+            if (config.AutomaticBaselineMode != AutomaticBaselineMode.None)
+            {
+                AutomaticBaselineMode = config.AutomaticBaselineMode;
+            }
         }
 
         /// <summary>

src/BenchmarkDotNet/Reports/Summary.cs

@@ -37,6 +37,7 @@ public class Summary
         private ImmutableDictionary<BenchmarkCase, BenchmarkReport> ReportMap { get; }
         private BaseliningStrategy BaseliningStrategy { get; }
         private bool? isMultipleRuntimes;
+        private readonly BenchmarkCase inferredBaselineBenchmarkCase;
 
         public Summary(
             string title,
@@ -62,13 +63,35 @@ public Summary(
             DisplayPrecisionManager = new DisplayPrecisionManager(this);
             Orderer = GetConfiguredOrdererOrDefaultOne(reports.Select(report => report.BenchmarkCase.Config));
             BenchmarksCases = Orderer.GetSummaryOrder(reports.Select(report => report.BenchmarkCase).ToImmutableArray(), this).ToImmutableArray(); // we sort it first
+            inferredBaselineBenchmarkCase = GetFastestBenchmarkCase(reports);
             Reports = BenchmarksCases.Select(b => ReportMap[b]).ToImmutableArray(); // we use sorted collection to re-create reports list
             BaseliningStrategy = BaseliningStrategy.Create(BenchmarksCases);
             Style = GetConfiguredSummaryStyleOrDefaultOne(BenchmarksCases).WithCultureInfo(cultureInfo);
             Table = GetTable(Style);
             AllRuntimes = BuildAllRuntimes(HostEnvironmentInfo, Reports);
         }
 
+        private static BenchmarkCase GetFastestBenchmarkCase(ImmutableArray<BenchmarkReport> reports)
+        {
+            if (reports.Any(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest))
+            {
+                var fastestReport = reports.First();
+                if (fastestReport.ResultStatistics != null)
+                {
+                    foreach (var report in reports.Skip(1).Where(r => r.ResultStatistics != null))
+                    {
+                        if (report.ResultStatistics.Mean < fastestReport.ResultStatistics.Mean)
+                        {
+                            fastestReport = report;
+                        }
+                    }
+                }
+                return fastestReport.BenchmarkCase;
+            }
+
+            return null;
+        }
+
         [PublicAPI] public bool HasReport(BenchmarkCase benchmarkCase) => ReportMap.ContainsKey(benchmarkCase);
 
         /// <summary>
@@ -133,7 +156,11 @@ public string GetLogicalGroupKey(BenchmarkCase benchmarkCase)
             => Orderer.GetLogicalGroupKey(BenchmarksCases, benchmarkCase);
 
         public bool IsBaseline(BenchmarkCase benchmarkCase)
-            => BaseliningStrategy.IsBaseline(benchmarkCase);
+        {
+            return inferredBaselineBenchmarkCase != null
+                ? inferredBaselineBenchmarkCase == benchmarkCase
+                : BaseliningStrategy.IsBaseline(benchmarkCase);
+        }
 
         [CanBeNull]
         public BenchmarkCase GetBaseline(string logicalGroupKey)

src/BenchmarkDotNet/Validators/BaselineValidator.cs

@@ -14,6 +14,17 @@ private BaselineValidator() { }
 
         public IEnumerable<ValidationError> Validate(ValidationParameters input)
         {
+            if (input.Config.AutomaticBaselineMode != Configs.AutomaticBaselineMode.None)
+            {
+                foreach (var benchmark in input.Benchmarks)
+                {
+                    if (benchmark.Descriptor.Baseline || benchmark.Job.Meta.Baseline)
+                    {
+                        yield return new ValidationError(TreatsWarningsAsErrors, "You cannot use both pre-configured and automatic baseline configuration", benchmark);
+                    }
+                }
+            }
+
             var allBenchmarks = input.Benchmarks.ToImmutableArray();
             var orderProvider = input.Config.Orderer;
 

tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs

@@ -0,0 +1,32 @@
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Tests.Mocks;
+using System;
+using System.Linq;
+using Xunit;
+
+namespace BenchmarkDotNet.IntegrationTests
+{
+    public class AutomaticBaselineTests
+    {
+        [Fact]
+        public void AutomaticBaselineSelectionIsCorrect()
+        {
+            var config = ManualConfig.CreateEmpty()
+                .AddColumnProvider(DefaultColumnProviders.Instance)
+                .WithAutomaticBaseline(AutomaticBaselineMode.Fastest);
+
+            var summary = MockFactory.CreateSummary(config, hugeSd: true, Array.Empty<Metric>());
+            var table = summary.GetTable(SummaryStyle.Default);
+            var method = table.Columns.Single(c => c.Header == "Method");
+            var ratio = table.Columns.Single(c => c.Header == "Ratio");
+
+            Assert.Equal(2, method.Content.Length);
+            Assert.Equal(nameof(MockFactory.MockBenchmarkClass.Foo), method.Content[0]);
+            Assert.Equal(1.0, double.Parse(ratio.Content[0])); // A faster one, see measurements in MockFactory.cs
+            Assert.Equal(nameof(MockFactory.MockBenchmarkClass.Bar), method.Content[1]);
+            Assert.Equal(1.5, double.Parse(ratio.Content[1]));
+        }
+    }
+}