From 2a37a2f3e8ff179d7dfaaa54cad9a25fdde5818d Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Sun, 16 Oct 2022 18:58:53 +0300 Subject: [PATCH 1/4] add tests --- ....GroupExporterTest.ParamArray.approved.txt | 19 +++++++ ...porterTest.ParamArrayOfArrays.approved.txt | 19 +++++++ .../MarkdownExporterApprovalTests.cs | 56 +++++++++++++++---- 3 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArray.approved.txt create mode 100644 tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArrayOfArrays.approved.txt diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArray.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArray.approved.txt new file mode 100644 index 0000000000..2a353ec410 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArray.approved.txt @@ -0,0 +1,19 @@ +=== ParamArray === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Method | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |-------- |---------:|--------:|--------:|------:|--------:|--------------------------------- |--------- | + Foo | Byte[1] | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=System.Byte[]]-DefaultJob | Yes | + Foo | Byte[1] | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=System.Byte[]]-DefaultJob | Yes | + Foo | Byte[1] | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | [Param=System.Byte[]]-DefaultJob | Yes | + Bar | Byte[1] | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | [Param=System.Byte[]]-DefaultJob | No | + Bar | Byte[1] | 502.0 ns | 6.09 ns | 1.58 ns | 4.92 | 0.06 | [Param=System.Byte[]]-DefaultJob | No | + Bar | Byte[1] | 602.0 ns | 6.09 ns | 1.58 ns | 5.90 | 0.08 | [Param=System.Byte[]]-DefaultJob | No | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArrayOfArrays.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArrayOfArrays.approved.txt new file mode 100644 index 0000000000..2ca356b167 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArrayOfArrays.approved.txt @@ -0,0 +1,19 @@ +=== ParamArrayOfArrays === + +BenchmarkDotNet=v0.10.x-mock, OS=Microsoft Windows NT 10.0.x.mock, VM=Hyper-V +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + + + Method | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |----------- |---------:|--------:|--------:|------:|--------:|------------------------------------ |--------- | + Foo | Int32[][2] | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=System.Int32[][]]-DefaultJob | Yes | + Foo | Int32[][2] | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=System.Int32[][]]-DefaultJob | Yes | + Foo | Int32[][2] | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | [Param=System.Int32[][]]-DefaultJob | Yes | + Bar | Int32[][2] | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | [Param=System.Int32[][]]-DefaultJob | No | + Bar | Int32[][2] | 502.0 ns | 6.09 ns | 1.58 ns | 4.92 | 0.06 | [Param=System.Int32[][]]-DefaultJob | No | + Bar | Int32[][2] | 602.0 ns | 6.09 ns | 1.58 ns | 5.90 | 0.08 | [Param=System.Int32[][]]-DefaultJob | No | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterApprovalTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterApprovalTests.cs index 9e502b192a..d2627991fa 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterApprovalTests.cs +++ b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterApprovalTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; @@ -225,8 +226,8 @@ [Benchmark] public void Bar() { } [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] public class MethodJobBaseline_MethodsJobs { - [Benchmark(Baseline = true)] public void Foo() {} - [Benchmark] public void Bar() {} + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } } [RankColumn, LogicalGroupColumn, BaselineColumn] @@ -235,8 +236,8 @@ public class MethodJobBaseline_MethodsJobsParams { [Params(2, 10), UsedImplicitly] public int Param; - [Benchmark(Baseline = true)] public void Foo() {} - [Benchmark] public void Bar() {} + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } } /* Invalid */ @@ -244,16 +245,16 @@ [Benchmark] public void Bar() {} [RankColumn, LogicalGroupColumn, BaselineColumn] public class Invalid_TwoMethodBaselines { - [Benchmark(Baseline = true)] public void Foo() {} - [Benchmark(Baseline = true)] public void Bar() {} + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark(Baseline = true)] public void Bar() { } } [RankColumn, LogicalGroupColumn, BaselineColumn] [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2", baseline: true)] public class Invalid_TwoJobBaselines { - [Benchmark] public void Foo() {} - [Benchmark] public void Bar() {} + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } } /* Escape Params */ @@ -262,10 +263,41 @@ public class Escape_ParamsAndArguments { [Params("\t", "\n"), UsedImplicitly] public string StringParam; - [Arguments('\t')] [Arguments('\n')] - [Benchmark] public void Foo(char charArg) {} - [Benchmark] public void Bar() {} + [Arguments('\t'), Arguments('\n')] + [Benchmark] public void Foo(char charArg) { } + [Benchmark] public void Bar() { } + } + + /* Param Arrays */ + + [BaselineColumn] + [LogicalGroupColumn] + public class ParamArray + { + [Params(new byte[] { 0 }, new byte[] { 1 }, new byte[] { 0 })] + public byte[] Param; + + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [BaselineColumn] + [LogicalGroupColumn] + public class ParamArrayOfArrays + { + [ParamsSource(nameof(GetValues))] + public int[][] Param; + + public IEnumerable GetValues() + { + yield return new int[][] { new[] { 0 }, new[] { 0 }, }; + yield return new int[][] { new[] { 0 }, new[] { 1 }, }; + yield return new int[][] { new[] { 0 }, new[] { 0 }, }; + } + + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } } } } -} +} \ No newline at end of file From 2366d72f515d8f827aa2c9ccbdfc9c006dfc8663 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Sun, 16 Oct 2022 19:41:46 +0300 Subject: [PATCH 2/4] Fix array comparison --- .../Parameters/ParameterComparer.cs | 22 +++++++++++++++++++ .../Parameters/ParameterInstances.cs | 18 ++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs index e5ae86d57b..5c058c7cfe 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs @@ -24,13 +24,35 @@ public int Compare(ParameterInstances x, ParameterInstances y) if (x == null) return -1; for (int i = 0; i < Math.Min(x.Count, y.Count); i++) { + //todo: compare non-primitive types too int compareTo = PrimitiveComparer.CompareTo(x[i]?.Value, y[i]?.Value); if (compareTo != 0) return compareTo; + + int arrayCompareTo = CompareArrays(x[i]?.Value, y[i]?.Value); + if (arrayCompareTo != 0) + return arrayCompareTo; } return string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo); } + private static int CompareArrays(object x, object y) + { + if (x is Array xArray && y is Array yArray) + { + for (int i = 0; i < Math.Min(xArray.Length, yArray.Length); i++) + { + //todo: compare non-primitive types too + int compareTo = PrimitiveComparer.CompareTo(xArray.GetValue(i), yArray.GetValue(i)); + if (compareTo != 0) + return compareTo; + } + if (xArray.Length != yArray.Length) + return xArray.Length.CompareTo(yArray.Length); + } + return 0; + } + private class Comparer { private readonly Dictionary> comparers = diff --git a/src/BenchmarkDotNet/Parameters/ParameterInstances.cs b/src/BenchmarkDotNet/Parameters/ParameterInstances.cs index 1f83b59310..86e6bb432f 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterInstances.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterInstances.cs @@ -31,10 +31,26 @@ public void Dispose() public string DisplayInfo => Items.Any() ? "[" + string.Join(", ", Items.Select(p => $"{p.Name}={p.ToDisplayText()}")) + "]" : ""; - public string ValueInfo => Items.Any() ? "[" + string.Join(", ", Items.Select(p => $"{p.Name}={p.Value?.ToString() ?? ParameterInstance.NullParameterTextRepresentation}")) + "]" : ""; + public string ValueInfo => Items.Any() ? "[" + string.Join(", ", Items.Select(p => $"{p.Name}={GetValueInfo(p.Value)}")) + "]" : ""; public string PrintInfo => printInfo ?? (printInfo = string.Join("&", Items.Select(p => $"{p.Name}={p.ToDisplayText()}"))); + private static string GetValueInfo(object value) + { + if (value is not Array array) + return value?.ToString() ?? "null"; + + //why it passes array values to cmd? + var strings = new List(array.Length); //test array of array + for (int i = 0; i < array.Length; i++) + { + string str = GetValueInfo(array.GetValue(i)); + strings.Add(str); + } + + return string.Join(",", strings); + } + public ParameterInstance GetArgument(string name) => Items.Single(parameter => parameter.IsArgument && parameter.Name == name); public bool Equals(ParameterInstances other) From 3f92bf9a3c5b20a2236f88141ae5d9d36310ecfe Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Sun, 16 Oct 2022 19:43:32 +0300 Subject: [PATCH 3/4] Update tests --- ...ts.GroupExporterTest.ParamArray.approved.txt | 17 +++++++++-------- ...ExporterTest.ParamArrayOfArrays.approved.txt | 17 +++++++++-------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArray.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArray.approved.txt index 2a353ec410..79e61f18c8 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArray.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArray.approved.txt @@ -7,13 +7,14 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Method | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | -------- |-------- |---------:|--------:|--------:|------:|--------:|--------------------------------- |--------- | - Foo | Byte[1] | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=System.Byte[]]-DefaultJob | Yes | - Foo | Byte[1] | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=System.Byte[]]-DefaultJob | Yes | - Foo | Byte[1] | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | [Param=System.Byte[]]-DefaultJob | Yes | - Bar | Byte[1] | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | [Param=System.Byte[]]-DefaultJob | No | - Bar | Byte[1] | 502.0 ns | 6.09 ns | 1.58 ns | 4.92 | 0.06 | [Param=System.Byte[]]-DefaultJob | No | - Bar | Byte[1] | 602.0 ns | 6.09 ns | 1.58 ns | 5.90 | 0.08 | [Param=System.Byte[]]-DefaultJob | No | + Method | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |-------- |---------:|--------:|--------:|------:|--------:|--------------------- |--------- | + Foo | Byte[1] | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=0]-DefaultJob | Yes | + Foo | Byte[1] | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=0]-DefaultJob | Yes | + Bar | Byte[1] | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | [Param=0]-DefaultJob | No | + Bar | Byte[1] | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | [Param=0]-DefaultJob | No | + | | | | | | | | | + Foo | Byte[1] | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=1]-DefaultJob | Yes | + Bar | Byte[1] | 602.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | [Param=1]-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArrayOfArrays.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArrayOfArrays.approved.txt index 2ca356b167..adc8e04c23 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArrayOfArrays.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.ParamArrayOfArrays.approved.txt @@ -7,13 +7,14 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC DefaultJob : extra output line - Method | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | -------- |----------- |---------:|--------:|--------:|------:|--------:|------------------------------------ |--------- | - Foo | Int32[][2] | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=System.Int32[][]]-DefaultJob | Yes | - Foo | Int32[][2] | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | [Param=System.Int32[][]]-DefaultJob | Yes | - Foo | Int32[][2] | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | [Param=System.Int32[][]]-DefaultJob | Yes | - Bar | Int32[][2] | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | [Param=System.Int32[][]]-DefaultJob | No | - Bar | Int32[][2] | 502.0 ns | 6.09 ns | 1.58 ns | 4.92 | 0.06 | [Param=System.Int32[][]]-DefaultJob | No | - Bar | Int32[][2] | 602.0 ns | 6.09 ns | 1.58 ns | 5.90 | 0.08 | [Param=System.Int32[][]]-DefaultJob | No | + Method | Param | Mean | Error | StdDev | Ratio | RatioSD | LogicalGroup | Baseline | +------- |----------- |---------:|--------:|--------:|------:|--------:|----------------------- |--------- | + Foo | Int32[][2] | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=0,0]-DefaultJob | Yes | + Foo | Int32[][2] | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | [Param=0,0]-DefaultJob | Yes | + Bar | Int32[][2] | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | [Param=0,0]-DefaultJob | No | + Bar | Int32[][2] | 602.0 ns | 6.09 ns | 1.58 ns | 5.90 | 0.08 | [Param=0,0]-DefaultJob | No | + | | | | | | | | | + Foo | Int32[][2] | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | [Param=0,1]-DefaultJob | Yes | + Bar | Int32[][2] | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | [Param=0,1]-DefaultJob | No | Errors: 0 From 2b06bbd9ea56ff8ddc84906689e1206d1a762cf0 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Mon, 17 Oct 2022 15:10:50 -0700 Subject: [PATCH 4/4] delete comment Co-authored-by: Adam Sitnik --- src/BenchmarkDotNet/Parameters/ParameterInstances.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/BenchmarkDotNet/Parameters/ParameterInstances.cs b/src/BenchmarkDotNet/Parameters/ParameterInstances.cs index 86e6bb432f..e97ea0cecf 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterInstances.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterInstances.cs @@ -40,7 +40,6 @@ private static string GetValueInfo(object value) if (value is not Array array) return value?.ToString() ?? "null"; - //why it passes array values to cmd? var strings = new List(array.Length); //test array of array for (int i = 0; i < array.Length; i++) {