Allow for Config per method, introduce OS and OSArchitecture filters (#1097)

Author: adamsitnik

samples/BenchmarkDotNet.Samples/IntroMemoryRandomization.cs

@@ -1,9 +1,5 @@
 using BenchmarkDotNet.Attributes;
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace BenchmarkDotNet.Samples
 {
@@ -23,6 +19,12 @@ public void Setup()
         }
 
         [Benchmark]
-        public void Array() => System.Array.Copy(_array, _destination, Size);
+        [MemoryRandomization(false)]
+        public void Array_RandomizationDisabled() => Array.Copy(_array, _destination, Size);
+
+        [Benchmark]
+        [MemoryRandomization(true)]
+        [MaxIterationCount(40)] // the benchmark becomes multimodal and need a lower limit of max iterations than the default
+        public void Array_RandomizationEnabled() => Array.Copy(_array, _destination, Size);
     }
 }

src/BenchmarkDotNet/Attributes/Filters/FilterConfigBaseAttribute.cs

@@ -4,7 +4,7 @@
 
 namespace BenchmarkDotNet.Attributes
 {
-    [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)]
+    [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)]
     public abstract class FilterConfigBaseAttribute : Attribute, IConfigSource
     {
         // CLS-Compliant Code requires a constructor without an array in the argument list

src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsArchitectureFilterAttribute.cs

@@ -0,0 +1,25 @@
+using System.Linq;
+using BenchmarkDotNet.Filters;
+using JetBrains.Annotations;
+using System.Runtime.InteropServices;
+
+namespace BenchmarkDotNet.Attributes
+{
+    [PublicAPI]
+    public class OperatingSystemsArchitectureFilterAttribute : FilterConfigBaseAttribute
+    {
+        // CLS-Compliant Code requires a constructor without an array in the argument list
+        public OperatingSystemsArchitectureFilterAttribute() { }
+
+        /// <param name="allowed">if set to true, the architectures are enabled, if set to false, disabled</param>
+        public OperatingSystemsArchitectureFilterAttribute(bool allowed, params Architecture[] architectures)
+            : base(new SimpleFilter(_ =>
+            {
+                return allowed
+                    ? architectures.Any(architecture => RuntimeInformation.OSArchitecture == architecture)
+                    : architectures.All(architecture => RuntimeInformation.OSArchitecture != architecture);
+            }))
+        {
+        }
+    }
+}

src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Linq;
+using BenchmarkDotNet.Filters;
+using JetBrains.Annotations;
+using System.Runtime.InteropServices;
+
+namespace BenchmarkDotNet.Attributes
+{
+    public enum OS : byte
+    {
+        Windows,
+        Linux,
+        macOS,
+        /// <summary>
+        /// WebAssembly
+        /// </summary>
+        Browser
+    }
+
+    [PublicAPI]
+    public class OperatingSystemsFilterAttribute : FilterConfigBaseAttribute
+    {
+        private static readonly OSPlatform browser = OSPlatform.Create("BROWSER");
+
+        // CLS-Compliant Code requires a constructor without an array in the argument list
+        public OperatingSystemsFilterAttribute() { }
+
+        /// <param name="allowed">if set to true, the OSes beloning to platforms are enabled, if set to false, disabled</param>
+        public OperatingSystemsFilterAttribute(bool allowed, params OS[] platforms)
+            : base(new SimpleFilter(_ =>
+            {
+                return allowed
+                    ? platforms.Any(platform => RuntimeInformation.IsOSPlatform(Map(platform)))
+                    : platforms.All(platform => !RuntimeInformation.IsOSPlatform(Map(platform)));
+            }))
+        {
+        }
+
+        // OSPlatform is a struct so it can not be used as attribute argument and this is why we use PlatformID enum
+        private static OSPlatform Map(OS platform)
+        {
+            switch (platform)
+            {
+                case OS.Windows:
+                    return OSPlatform.Windows;
+                case OS.Linux:
+                    return OSPlatform.Linux;
+                case OS.macOS:
+                    return OSPlatform.OSX;
+                case OS.Browser:
+                    return browser;
+                default:
+                    throw new NotSupportedException($"Platform {platform} is not supported");
+            }
+        }
+    }
+}

src/BenchmarkDotNet/Attributes/Mutators/JobMutatorConfigBaseAttribute.cs

@@ -5,7 +5,7 @@
 
 namespace BenchmarkDotNet.Attributes
 {
-    [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] // users must not be able to define given mutator attribute more than once per type
+    [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = false)] // users must not be able to define given mutator attribute more than once per type
     public class JobMutatorConfigBaseAttribute : Attribute, IConfigSource
     {
         // CLS-Compliant Code requires a constructor which use only CLS-compliant types

src/BenchmarkDotNet/Running/BenchmarkConverter.cs

@@ -23,78 +23,78 @@ public static BenchmarkRunInfo TypeToBenchmarks(Type type, IConfig config = null
 
             // We should check all methods including private to notify users about private methods with the [Benchmark] attribute
             var bindingFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
+            var benchmarkMethods = type.GetMethods(bindingFlags).Where(method => method.HasAttribute<BenchmarkAttribute>()).ToArray();
 
-            var fullConfig = GetFullConfig(type, config);
-            var allMethods = type.GetMethods(bindingFlags);
-            return MethodsToBenchmarksWithFullConfig(type, allMethods, fullConfig);
+            return MethodsToBenchmarksWithFullConfig(type, benchmarkMethods, config);
         }
 
         public static BenchmarkRunInfo MethodsToBenchmarks(Type containingType, MethodInfo[] benchmarkMethods, IConfig config = null)
-        {
-            var fullConfig = GetFullConfig(containingType, config);
-
-            return MethodsToBenchmarksWithFullConfig(containingType, benchmarkMethods, fullConfig);
-        }
+            => MethodsToBenchmarksWithFullConfig(containingType, benchmarkMethods, config);
 
-        private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type containingType, MethodInfo[] benchmarkMethods, ImmutableConfig immutableConfig)
+        private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type type, MethodInfo[] benchmarkMethods, IConfig config)
         {
-            if (immutableConfig == null)
-                throw new ArgumentNullException(nameof(immutableConfig));
-
-            var helperMethods = containingType.GetMethods(); // benchmarkMethods can be filtered, without Setups, look #564
-
-            var globalSetupMethods = GetAttributedMethods<GlobalSetupAttribute>(helperMethods, "GlobalSetup");
-            var globalCleanupMethods = GetAttributedMethods<GlobalCleanupAttribute>(helperMethods, "GlobalCleanup");
-            var iterationSetupMethods = GetAttributedMethods<IterationSetupAttribute>(helperMethods, "IterationSetup");
-            var iterationCleanupMethods = GetAttributedMethods<IterationCleanupAttribute>(helperMethods, "IterationCleanup");
-
-            var targetMethods = benchmarkMethods.Where(method => method.HasAttribute<BenchmarkAttribute>()).ToArray();
+            var allPublicMethods = type.GetMethods(); // benchmarkMethods can be filtered, without Setups, look #564
+            var configPerType = GetFullTypeConfig(type, config);
 
-            var parameterDefinitions = GetParameterDefinitions(containingType);
-            var parameterInstancesList = parameterDefinitions.Expand(immutableConfig.SummaryStyle);
+            var globalSetupMethods = GetAttributedMethods<GlobalSetupAttribute>(allPublicMethods, "GlobalSetup");
+            var globalCleanupMethods = GetAttributedMethods<GlobalCleanupAttribute>(allPublicMethods, "GlobalCleanup");
+            var iterationSetupMethods = GetAttributedMethods<IterationSetupAttribute>(allPublicMethods, "IterationSetup");
+            var iterationCleanupMethods = GetAttributedMethods<IterationCleanupAttribute>(allPublicMethods, "IterationCleanup");
 
-            var jobs = immutableConfig.GetJobs();
+            var targets = GetTargets(benchmarkMethods, type, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods).ToArray();
 
-            var targets = GetTargets(targetMethods, containingType, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods).ToArray();
+            var parameterDefinitions = GetParameterDefinitions(type);
+            var parameterInstancesList = parameterDefinitions.Expand(configPerType.SummaryStyle);
 
             var benchmarks = new List<BenchmarkCase>();
+
             foreach (var target in targets)
             {
-                var argumentsDefinitions = GetArgumentsDefinitions(target.WorkloadMethod, target.Type, immutableConfig.SummaryStyle).ToArray();
+                var argumentsDefinitions = GetArgumentsDefinitions(target.WorkloadMethod, target.Type, configPerType.SummaryStyle).ToArray();
 
                 var parameterInstances =
                     (from parameterInstance in parameterInstancesList
                      from argumentDefinition in argumentsDefinitions
                      select new ParameterInstances(parameterInstance.Items.Concat(argumentDefinition.Items).ToArray())).ToArray();
 
-                benchmarks.AddRange(
-                    from job in jobs
+                var configPerMethod = GetFullMethodConfig(target.WorkloadMethod, configPerType);
+
+                var benchmarksForTarget =
+                    from job in configPerMethod.GetJobs()
                     from parameterInstance in parameterInstances
-                    select BenchmarkCase.Create(target, job, parameterInstance, immutableConfig)
-                );
+                    select BenchmarkCase.Create(target, job, parameterInstance, configPerMethod);
+
+                benchmarks.AddRange(GetFilteredBenchmarks(benchmarksForTarget, configPerMethod.GetFilters()));
             }
 
-            var filters = immutableConfig.GetFilters().ToArray();
-            var filteredBenchmarks = GetFilteredBenchmarks(benchmarks, filters);
-            var orderedBenchmarks = immutableConfig.Orderer.GetExecutionOrder(filteredBenchmarks).ToArray();
+            var orderedBenchmarks = configPerType.Orderer.GetExecutionOrder(benchmarks.ToImmutableArray()).ToArray();
 
-            return new BenchmarkRunInfo(orderedBenchmarks, containingType, immutableConfig);
+            return new BenchmarkRunInfo(orderedBenchmarks, type, configPerType);
         }
 
-        public static ImmutableConfig GetFullConfig(Type type, IConfig config)
+        private static ImmutableConfig GetFullTypeConfig(Type type, IConfig config)
         {
             config = config ?? DefaultConfig.Instance;
-            if (type != null)
-            {
-                var typeAttributes = type.GetTypeInfo().GetCustomAttributes(true).OfType<IConfigSource>();
-                var assemblyAttributes = type.GetTypeInfo().Assembly.GetCustomAttributes().OfType<IConfigSource>();
-                var allAttributes = typeAttributes.Concat(assemblyAttributes);
-                var configs = allAttributes.Select(attribute => attribute.Config)
-                    .OrderBy(c => c.GetJobs().Count(job => job.Meta.IsMutator)); // configs with mutators must be the ones applied at the end
-
-                foreach (var configFromAttribute in configs)
-                    config = ManualConfig.Union(config, configFromAttribute);
-            }
+
+            var typeAttributes = type.GetCustomAttributes(true).OfType<IConfigSource>();
+            var assemblyAttributes = type.Assembly.GetCustomAttributes().OfType<IConfigSource>();
+
+            foreach (var configFromAttribute in typeAttributes.Concat(assemblyAttributes))
+                config = ManualConfig.Union(config, configFromAttribute.Config);
+
+            return ImmutableConfigBuilder.Create(config);
+        }
+
+        private static ImmutableConfig GetFullMethodConfig(MethodInfo method, ImmutableConfig typeConfig)
+        {
+            var methodAttributes = method.GetCustomAttributes(true).OfType<IConfigSource>();
+
+            if (!methodAttributes.Any()) // the most common case
+                return typeConfig;
+
+            var config = ManualConfig.Create(typeConfig);
+            foreach (var configFromAttribute in methodAttributes)
+                config = ManualConfig.Union(config, configFromAttribute.Config);
 
             return ImmutableConfigBuilder.Create(config);
         }
@@ -251,7 +251,7 @@ private static string[] GetCategories(MethodInfo method)
             return attributes.SelectMany(attr => attr.Categories).Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
         }
 
-        private static ImmutableArray<BenchmarkCase> GetFilteredBenchmarks(IList<BenchmarkCase> benchmarks, IList<IFilter> filters)
+        private static ImmutableArray<BenchmarkCase> GetFilteredBenchmarks(IEnumerable<BenchmarkCase> benchmarks, IEnumerable<IFilter> filters)
             => benchmarks.Where(benchmark => filters.All(filter => filter.Predicate(benchmark))).ToImmutableArray();
 
         private static void AssertMethodHasCorrectSignature(string methodType, MethodInfo methodInfo)

src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs

@@ -13,7 +13,7 @@ public static class BenchmarkPartitioner
     {
         public static BuildPartition[] CreateForBuild(BenchmarkRunInfo[] supportedBenchmarks, IResolver resolver)
             => supportedBenchmarks
-                .SelectMany(info => info.BenchmarksCases.Select(benchmark => (benchmark, info.Config)))
+                .SelectMany(info => info.BenchmarksCases.Select(benchmark => (benchmark, benchmark.Config)))
                 .GroupBy(tuple => tuple.benchmark, BenchmarkRuntimePropertiesComparer.Instance)
                 .Select(group => new BuildPartition(group.Select((item, index) => new BenchmarkBuildInfo(item.benchmark, item.Config, index)).ToArray(), resolver))
                 .ToArray();

tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs

@@ -60,7 +60,7 @@ protected Reports.Summary CanExecute(Type type, IConfig config = null, bool full
                 config = config.AddColumnProvider(DefaultColumnProviders.Instance);
 
             // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class
-            var summary = BenchmarkRunner.Run(type, BenchmarkConverter.GetFullConfig(type, config));
+            var summary = BenchmarkRunner.Run(type, config);
 
             if (fullValidation)
             {