Skip to content

Issue #1024: Calculate baseline by the fastest benchmark #2171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Issue #1024: Calculate baseline by the fastest benchmark #2171

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
deedc64
Issue #1024: Calculate baseline by the fastest benchmark
Serg046 Oct 26, 2022
fd9f34f
Issue #1024: Add validations
Serg046 Oct 26, 2022
d20c117
Issue #1024: Correct configuration
Serg046 Oct 30, 2022
d8d99e2
Issue #1024: Make AutomaticBaselineSelectionIsCorrect test stable enough
Serg046 Oct 31, 2022
873243d
Update src/BenchmarkDotNet/Reports/Summary.cs
Serg046 Oct 31, 2022
885bc04
Fix a typo
Serg046 Oct 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/BenchmarkDotNet/Attributes/AutomaticBaselineAttribute.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
using BenchmarkDotNet.Configs;
using System;

namespace BenchmarkDotNet.Attributes
{
[AttributeUsage(AttributeTargets.Class)]
public class AutomaticBaselineAttribute : Attribute, IConfigSource
{
public IConfig Config { get; }

public AutomaticBaselineAttribute(AutomaticBaselineMode mode) => Config = ManualConfig.CreateEmpty().WithAutomaticBaseline(mode);
}
}
8 changes: 8 additions & 0 deletions src/BenchmarkDotNet/Configs/AutomaticBaselineMode.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
namespace BenchmarkDotNet.Configs
{
public enum AutomaticBaselineMode
{
None,
Fastest
}
}
2 changes: 2 additions & 0 deletions src/BenchmarkDotNet/Configs/DebugConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,5 +87,7 @@ public string ArtifactsPath
public ConfigOptions Options => ConfigOptions.KeepBenchmarkFiles | ConfigOptions.DisableOptimizationsValidator;

public IReadOnlyList<Conclusion> ConfigAnalysisConclusion => emptyConclusion;

public AutomaticBaselineMode AutomaticBaselineMode { get; } = AutomaticBaselineMode.None;
}
}
2 changes: 2 additions & 0 deletions src/BenchmarkDotNet/Configs/DefaultConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,5 +106,7 @@ public string ArtifactsPath
public IEnumerable<IFilter> GetFilters() => Array.Empty<IFilter>();

public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => Array.Empty<IColumnHidingRule>();

public AutomaticBaselineMode AutomaticBaselineMode { get; } = AutomaticBaselineMode.None;
}
}
2 changes: 2 additions & 0 deletions src/BenchmarkDotNet/Configs/IConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,5 +57,7 @@ public interface IConfig
/// Collect any errors or warnings when composing the configuration
/// </summary>
IReadOnlyList<Conclusion> ConfigAnalysisConclusion { get; }

AutomaticBaselineMode AutomaticBaselineMode { get; }
}
}
5 changes: 4 additions & 1 deletion src/BenchmarkDotNet/Configs/ImmutableConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ internal ImmutableConfig(
SummaryStyle summaryStyle,
ConfigOptions options,
TimeSpan buildTimeout,
IReadOnlyList<Conclusion> configAnalysisConclusion)
IReadOnlyList<Conclusion> configAnalysisConclusion,
AutomaticBaselineMode automaticBaselineMode)
{
columnProviders = uniqueColumnProviders;
loggers = uniqueLoggers;
Expand All @@ -74,6 +75,7 @@ internal ImmutableConfig(
Options = options;
BuildTimeout = buildTimeout;
ConfigAnalysisConclusion = configAnalysisConclusion;
AutomaticBaselineMode = automaticBaselineMode;
}

public ConfigUnionRule UnionRule { get; }
Expand All @@ -83,6 +85,7 @@ internal ImmutableConfig(
[NotNull] public IOrderer Orderer { get; }
public SummaryStyle SummaryStyle { get; }
public TimeSpan BuildTimeout { get; }
public AutomaticBaselineMode AutomaticBaselineMode { get; }

public IEnumerable<IColumnProvider> GetColumnProviders() => columnProviders;
public IEnumerable<IExporter> GetExporters() => exporters;
Expand Down
3 changes: 2 additions & 1 deletion src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,8 @@ public static ImmutableConfig Create(IConfig source)
source.SummaryStyle ?? SummaryStyle.Default,
source.Options,
source.BuildTimeout,
configAnalyse.AsReadOnly()
configAnalyse.AsReadOnly(),
source.AutomaticBaselineMode
);
}

Expand Down
8 changes: 8 additions & 0 deletions src/BenchmarkDotNet/Configs/ManualConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ public class ManualConfig : IConfig
[PublicAPI] public IOrderer Orderer { get; set; }
[PublicAPI] public SummaryStyle SummaryStyle { get; set; }
[PublicAPI] public TimeSpan BuildTimeout { get; set; } = DefaultConfig.Instance.BuildTimeout;
[PublicAPI] public AutomaticBaselineMode AutomaticBaselineMode { get; private set; }

public IReadOnlyList<Conclusion> ConfigAnalysisConclusion => emptyConclusion;

Expand Down Expand Up @@ -98,6 +99,12 @@ public ManualConfig WithBuildTimeout(TimeSpan buildTimeout)
return this;
}

public ManualConfig WithAutomaticBaseline(AutomaticBaselineMode automaticBaselineMode)
{
AutomaticBaselineMode = automaticBaselineMode;
return this;
}

[EditorBrowsable(EditorBrowsableState.Never)]
[Obsolete("This method will soon be removed, please start using .AddColumn() instead.")]
public void Add(params IColumn[] newColumns) => AddColumn(newColumns);
Expand Down Expand Up @@ -254,6 +261,7 @@ public void Add(IConfig config)
columnHidingRules.AddRange(config.GetColumnHidingRules());
Options |= config.Options;
BuildTimeout = GetBuildTimeout(BuildTimeout, config.BuildTimeout);
AutomaticBaselineMode = config.AutomaticBaselineMode;
}

/// <summary>
Expand Down
29 changes: 28 additions & 1 deletion src/BenchmarkDotNet/Reports/Summary.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public class Summary
private ImmutableDictionary<BenchmarkCase, BenchmarkReport> ReportMap { get; }
private BaseliningStrategy BaseliningStrategy { get; }
private bool? isMultipleRuntimes;
private readonly BenchmarkCase inferredBaselineBenchmarkCase;

public Summary(
string title,
Expand All @@ -62,13 +63,35 @@ public Summary(
DisplayPrecisionManager = new DisplayPrecisionManager(this);
Orderer = GetConfiguredOrdererOrDefaultOne(reports.Select(report => report.BenchmarkCase.Config));
BenchmarksCases = Orderer.GetSummaryOrder(reports.Select(report => report.BenchmarkCase).ToImmutableArray(), this).ToImmutableArray(); // we sort it first
inferredBaselineBenchmarkCase = GetFastestBenchmarkCase(reports);
Reports = BenchmarksCases.Select(b => ReportMap[b]).ToImmutableArray(); // we use sorted collection to re-create reports list
BaseliningStrategy = BaseliningStrategy.Create(BenchmarksCases);
Style = GetConfiguredSummaryStyleOrDefaultOne(BenchmarksCases).WithCultureInfo(cultureInfo);
Table = GetTable(Style);
AllRuntimes = BuildAllRuntimes(HostEnvironmentInfo, Reports);
}

private static BenchmarkCase GetFastestBenchmarkCase(ImmutableArray<BenchmarkReport> reports)
{
if (reports.Any() && reports.All(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

reports.All(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest))

Is it correctly works when 2 benchmark classes have different AutomaticBaselineMode?

[AutomaticBaseline(AutomaticBaselineMode.Fastest)]
public class C1 { }

[AutomaticBaseline(AutomaticBaselineMode.None)]
public class C2 { }

To join summaries, use:

  1. --join (for cmd)
  2. BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).RunAllJoined()

Copy link
Contributor Author

@Serg046 Serg046 Oct 26, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The idea itself works because the benchmarks are already grouped by the type so that Summary contains the cases of either C1 or C2 only. However, this particular example doesn't print the ratio for all the methods. I guess BDN trims columns that don't exist in all results (C1's benchmarks contain ration but C2's ones don't). If you set AutomaticBaselineMode.Fastest for C2, it prints correctly.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ManualConfig.Union(...) resets the AutomaticBaselineMode:

config = ManualConfig.Union(config, configFromAttribute.Config) // C1, set the value to "Fastest"
config = ManualConfig.Union(config, configFromAttribute.Config) // C2, set the value to "None" (even if the `[AutomaticBaseline(None)]` is not used)

The problem is that the final config has only one value of AutomaticBaselineMode, but it should have a different value for each benchmark class to allow:

[AutomaticBaseline(AutomaticBaselineMode.Fastest)] public class B1 { }
public class B2 { } //implicit AutomaticBaselineMode.None
[AutomaticBaseline(AutomaticBaselineMode.Slowest)] public class B3 { }

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, it is not like that, the configs are different. I see that this inferredBaselineBenchmarkCase is calculated properly. There is probably a need to add a column or something like that, I will try to identify the problem.

Copy link
Contributor

@YegorStepanov YegorStepanov Oct 28, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, it is like that :)

Apply it first:

Suggested change
if (reports.Any() && reports.All(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest))
if (reports.Any() && reports.Any(r => r.BenchmarkCase.Config.AutomaticBaselineMode == AutomaticBaselineMode.Fastest))

Sample

[DryJob]
// [AutomaticBaseline(AutomaticBaselineMode.Fastest)]
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] // it applies to all summaries, we may not duplicate it
[BenchmarkCategory("A")] // without this, the table is not split by classes
public class MyClass1
{
    [Benchmark] public void MyMethod1() => Thread.Sleep(100);
    [Benchmark] public void MyMethod2() => Thread.Sleep(200);
}

[DryJob]
// [AutomaticBaseline(AutomaticBaselineMode.Fastest)]
[BenchmarkCategory("B")]
public class MyClass2
{
    [Benchmark] public void MyMethod1() => Thread.Sleep(400);
    [Benchmark] public void MyMethod2() => Thread.Sleep(200);
}

If we uncomment AutomaticBaseline for MyClass1 only:

image

If we uncomment AutomaticBaseline for MyClass2 only (this is the correct table):

image

Debug these lines to see how IConfig.AutomaticBaselineMode changes:

BenchmarkDotNet/src/BenchmarkDotNet/Running/BenchmarkConverter.cs

Lines 106 to 107 in 6bb61ce

foreach (var configFromAttribute in methodAttributes)
config = ManualConfig.Union(config, configFromAttribute.Config);

Copy link
Contributor Author

@Serg046 Serg046 Oct 30, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, you are right. I didn't notice that Summary is built the third time (where you have all 4 methods). I was checking first two Summary instances (first two benchmarks, then second two). Thank you!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, this is what I suggest we do d20c117

  1. We skip all defaut (None) values to build configs.
  2. We take the latest non-default value to determine whether we need to override baseline functionality or we don't.

{
var fastestReport = reports.First();
if (fastestReport.ResultStatistics != null)
{
foreach (var report in reports.Skip(1).Where(r => r.ResultStatistics != null))
{
if (report.ResultStatistics.Mean < fastestReport.ResultStatistics.Mean)
{
fastestReport = report;
}
}
}
return fastestReport.BenchmarkCase;
}

return null;
}

[PublicAPI] public bool HasReport(BenchmarkCase benchmarkCase) => ReportMap.ContainsKey(benchmarkCase);

/// <summary>
Expand Down Expand Up @@ -133,7 +156,11 @@ public string GetLogicalGroupKey(BenchmarkCase benchmarkCase)
=> Orderer.GetLogicalGroupKey(BenchmarksCases, benchmarkCase);

public bool IsBaseline(BenchmarkCase benchmarkCase)
=> BaseliningStrategy.IsBaseline(benchmarkCase);
{
return inferredBaselineBenchmarkCase != null
? inferredBaselineBenchmarkCase == benchmarkCase
: BaseliningStrategy.IsBaseline(benchmarkCase);
}

[CanBeNull]
public BenchmarkCase GetBaseline(string logicalGroupKey)
Expand Down
11 changes: 11 additions & 0 deletions src/BenchmarkDotNet/Validators/BaselineValidator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,17 @@ private BaselineValidator() { }

public IEnumerable<ValidationError> Validate(ValidationParameters input)
{
if (input.Config.AutomaticBaselineMode != Configs.AutomaticBaselineMode.None)
{
foreach (var benchmark in input.Benchmarks)
{
if (benchmark.Descriptor.Baseline || benchmark.Job.Meta.Baseline)
{
yield return new ValidationError(TreatsWarningsAsErrors, "You cannot use both pre-configured and automatic baseline configuration", benchmark);
}
}
}

var allBenchmarks = input.Benchmarks.ToImmutableArray();
var orderProvider = input.Config.Orderer;

Expand Down
53 changes: 53 additions & 0 deletions tests/BenchmarkDotNet.IntegrationTests/AutomaticBaselineTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
using System.Linq;
using System.Threading;
using Xunit;

namespace BenchmarkDotNet.IntegrationTests
{
public class AutomaticBaselineTests
{
[Fact]
public void AutomaticBaselineSelectionIsCorrect()
{
var config = CreateConfig();

var summary = BenchmarkRunner.Run<BaselineSample>();

var table = summary.GetTable(SummaryStyle.Default);
var column = table.Columns.Single(c => c.Header == "Ratio");
Assert.Equal(2, column.Content.Length);
Assert.Equal(1.0, double.Parse(column.Content[1])); // Ratio of TwoMilliseconds
Assert.True(double.Parse(column.Content[0]) > 1.0); // Ratio of TwoHundredMilliseconds
}

[AutomaticBaseline(AutomaticBaselineMode.Fastest)]
public class BaselineSample
{
[Benchmark]
public void TwoHundredMilliseconds()
{
Thread.Sleep(200);
}

[Benchmark]
public void TwoMilliseconds()
{
Thread.Sleep(2);
}
}

private IConfig CreateConfig()
=> ManualConfig.CreateEmpty()
.AddJob(Job.ShortRun
.WithEvaluateOverhead(false) // no need to run idle for this test
.WithWarmupCount(0) // don't run warmup to save some time for our CI runs
.WithIterationCount(1)) // single iteration is enough for us
.AddColumnProvider(DefaultColumnProviders.Instance);
}
}