Allow for Config per method, introduce OS and OSArchitecture filters

src/BenchmarkDotNet/Attributes/Filters/FilterConfigBaseAttribute.cs

@@ -4,7 +4,7 @@
 
 namespace BenchmarkDotNet.Attributes
 {    
-    [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)]
+    [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)]
     public abstract class FilterConfigBaseAttribute : Attribute, IConfigSource
     {
         // CLS-Compliant Code requires a constructor without an array in the argument list

src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsArchitectureFilterAttribute.cs

@@ -0,0 +1,25 @@
+using System.Linq;
+using BenchmarkDotNet.Filters;
+using JetBrains.Annotations;
+using System.Runtime.InteropServices;
+
+namespace BenchmarkDotNet.Attributes
+{
+    [PublicAPI]
+    public class OperatingSystemsArchitectureFilterAttribute : FilterConfigBaseAttribute
+    {
+        // CLS-Compliant Code requires a constructor without an array in the argument list
+        public OperatingSystemsArchitectureFilterAttribute() { }
+
+        /// <param name="allowed">if set to true, the architectures are enabled, if set to false, disabled</param>
+        public OperatingSystemsArchitectureFilterAttribute(bool allowed, params Architecture[] architectures)
+            : base(new SimpleFilter(_ =>
+            {
+                return allowed
+                    ? architectures.Any(architecture => RuntimeInformation.OSArchitecture == architecture)
+                    : architectures.All(architecture => RuntimeInformation.OSArchitecture != architecture);
+            }))
+        {
+        }
+    }
+}

src/BenchmarkDotNet/Attributes/Filters/OperatingSystemsFilterAttribute.cs

@@ -0,0 +1,45 @@
+using System;
+using System.Linq;
+using BenchmarkDotNet.Filters;
+using JetBrains.Annotations;
+using System.Runtime.InteropServices;
+
+namespace BenchmarkDotNet.Attributes
+{
+    [PublicAPI]
+    public class OperatingSystemsFilterAttribute : FilterConfigBaseAttribute
+    {
+        // CLS-Compliant Code requires a constructor without an array in the argument list
+        public OperatingSystemsFilterAttribute() { }
+
+        /// <param name="allowed">if set to true, the OSes beloning to platforms are enabled, if set to false, disabled</param>
+        public OperatingSystemsFilterAttribute(bool allowed, params PlatformID[] platforms)
+            : base(new SimpleFilter(_ =>
+            {
+                return allowed
+                    ? platforms.Any(platform => RuntimeInformation.IsOSPlatform(Map(platform)))
+                    : platforms.All(platform => !RuntimeInformation.IsOSPlatform(Map(platform)));
+            }))
+        {
+        }
+
+        // OSPlatform is a struct so it can not be used as attribute argument and this is why we use PlatformID enum
+        private static OSPlatform Map(PlatformID platform)
+        {
+            switch (platform)
+            {
+                case PlatformID.MacOSX:
+                    return OSPlatform.OSX;
+                case PlatformID.Unix:
+                    return OSPlatform.Linux;
+                case PlatformID.Win32NT:
+                case PlatformID.Win32S:
+                case PlatformID.Win32Windows:
+                case PlatformID.WinCE:
+                    return OSPlatform.Windows;
+                default:
+                    throw new NotSupportedException($"Platform {platform} is not supported");
+            }
+        }
+    }
+}

src/BenchmarkDotNet/Running/BenchmarkConverter.cs

@@ -9,6 +9,7 @@
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Filters;
+using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Parameters;
 
 namespace BenchmarkDotNet.Running
@@ -22,41 +23,24 @@ public static BenchmarkRunInfo TypeToBenchmarks(Type type, IConfig config = null
 
             // We should check all methods including private to notify users about private methods with the [Benchmark] attribute
             var bindingFlags = BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
-
-            var fullConfig = GetFullConfig(type, config);
-            var allMethods = type.GetMethods(bindingFlags);
-            return MethodsToBenchmarksWithFullConfig(type, allMethods, fullConfig);
-        }
-
-        public static BenchmarkRunInfo MethodsToBenchmarks(Type containingType, MethodInfo[] benchmarkMethods, IConfig config = null)
-        {
-            var fullConfig = GetFullConfig(containingType, config);
-
-            return MethodsToBenchmarksWithFullConfig(containingType, benchmarkMethods, fullConfig);
-        }
+            var benchmarkMethods = type.GetMethods(bindingFlags).Where(method => method.HasAttribute<BenchmarkAttribute>()).ToArray();
 
-        private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type containingType, MethodInfo[] benchmarkMethods, ImmutableConfig immutableConfig)
-        {
-            if (immutableConfig == null)
-                throw new ArgumentNullException(nameof(immutableConfig));
-
-            var helperMethods = containingType.GetMethods(); // benchmarkMethods can be filtered, without Setups, look #564
-
-            var globalSetupMethods = GetAttributedMethods<GlobalSetupAttribute>(helperMethods, "GlobalSetup");
-            var globalCleanupMethods = GetAttributedMethods<GlobalCleanupAttribute>(helperMethods, "GlobalCleanup");
-            var iterationSetupMethods = GetAttributedMethods<IterationSetupAttribute>(helperMethods, "IterationSetup");
-            var iterationCleanupMethods = GetAttributedMethods<IterationCleanupAttribute>(helperMethods, "IterationCleanup");
+            var allPublicMethods = type.GetMethods(); // benchmarkMethods can be filtered, without Setups, look #564
 
-            var targetMethods = benchmarkMethods.Where(method => method.HasAttribute<BenchmarkAttribute>()).ToArray();
+            var globalSetupMethods = GetAttributedMethods<GlobalSetupAttribute>(allPublicMethods, "GlobalSetup");
+            var globalCleanupMethods = GetAttributedMethods<GlobalCleanupAttribute>(allPublicMethods, "GlobalCleanup");
+            var iterationSetupMethods = GetAttributedMethods<IterationSetupAttribute>(allPublicMethods, "IterationSetup");
+            var iterationCleanupMethods = GetAttributedMethods<IterationCleanupAttribute>(allPublicMethods, "IterationCleanup");
 
-            var parameterDefinitions = GetParameterDefinitions(containingType);
+            var parameterDefinitions = GetParameterDefinitions(type);
             var parameterInstancesList = parameterDefinitions.Expand();
 
-            var jobs = immutableConfig.GetJobs();
+            var targets = GetTargets(benchmarkMethods, type, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods).ToArray();
 
-            var targets = GetTargets(targetMethods, containingType, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods).ToArray();
+            var configPerType = GetFullTypeConfig(type, config);
 
             var benchmarks = new List<BenchmarkCase>();
+
             foreach (var target in targets)
             {
                 var argumentsDefinitions = GetArgumentsDefinitions(target.WorkloadMethod, target.Type).ToArray();
@@ -66,38 +50,48 @@ private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type containin
                      from argumentDefinition in argumentsDefinitions
                      select new ParameterInstances(parameterInstance.Items.Concat(argumentDefinition.Items).ToArray())).ToArray();
 
-                benchmarks.AddRange(
-                    from job in jobs
+                var configPerMethod = GetFullMethodConfig(target.WorkloadMethod, configPerType);
+
+                var benchmarksForTarget = 
+                    from job in configPerMethod.GetJobs()
                     from parameterInstance in parameterInstances
-                    select BenchmarkCase.Create(target, job, parameterInstance, immutableConfig)
-                );
+                    select BenchmarkCase.Create(target, job, parameterInstance, configPerMethod);
+
+                benchmarks.AddRange(GetFilteredBenchmarks(benchmarksForTarget, configPerMethod.GetFilters()));
             }
 
-            var filters = immutableConfig.GetFilters().ToArray();
-            var filteredBenchmarks = GetFilteredBenchmarks(benchmarks, filters);
-            var orderedBenchmarks = immutableConfig.Orderer.GetExecutionOrder(filteredBenchmarks).ToArray();
+            var orderedBenchmarks = configPerType.Orderer.GetExecutionOrder(benchmarks.ToImmutableArray()).ToArray();
 
-            return new BenchmarkRunInfo(orderedBenchmarks, containingType, immutableConfig);
+            return new BenchmarkRunInfo(orderedBenchmarks, type, configPerType);
         }
 
-        public static ImmutableConfig GetFullConfig(Type type, IConfig config)
+        private static ImmutableConfig GetFullTypeConfig(Type type, IConfig config)
         {
             config = config ?? DefaultConfig.Instance;
-            if (type != null)
-            {
-                var typeAttributes = type.GetTypeInfo().GetCustomAttributes(true).OfType<IConfigSource>();
-                var assemblyAttributes = type.GetTypeInfo().Assembly.GetCustomAttributes().OfType<IConfigSource>();
-                var allAttributes = typeAttributes.Concat(assemblyAttributes);
-                var configs = allAttributes.Select(attribute => attribute.Config)
-                    .OrderBy(c => c.GetJobs().Count(job => job.Meta.IsMutator)); // configs with mutators must be the ones applied at the end
-
-                foreach (var configFromAttribute in configs)
-                    config = ManualConfig.Union(config, configFromAttribute);
-            }
+
+            var typeAttributes = type.GetCustomAttributes(true).OfType<IConfigSource>();
+            var assemblyAttributes = type.Assembly.GetCustomAttributes().OfType<IConfigSource>();
+
+            foreach (var configFromAttribute in typeAttributes.Concat(assemblyAttributes))
+                config = ManualConfig.Union(config, configFromAttribute.Config);
 
             return ImmutableConfigBuilder.Create(config);
         }
 
+        private static ImmutableConfig GetFullMethodConfig(MethodInfo method, ImmutableConfig typeConfig)
+        {
+            var methodAttributes = method.GetCustomAttributes(true).OfType<IConfigSource>();
+
+            if (!methodAttributes.Any()) // the most common case
+                return typeConfig;
+
+            var config = ManualConfig.Create(typeConfig);
+            foreach (var configFromAttribute in methodAttributes)
+                config = ManualConfig.Union(config, configFromAttribute.Config);
+
+            return ImmutableConfigBuilder.Create(config);
+        }
+
         private static IEnumerable<Descriptor> GetTargets(
             MethodInfo[] targetMethods,
             Type type,
@@ -238,7 +232,7 @@ private static string[] GetCategories(MethodInfo method)
             return attributes.SelectMany(attr => attr.Categories).Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
         }
 
-        private static ImmutableArray<BenchmarkCase> GetFilteredBenchmarks(IList<BenchmarkCase> benchmarks, IList<IFilter> filters) 
+        private static ImmutableArray<BenchmarkCase> GetFilteredBenchmarks(IEnumerable<BenchmarkCase> benchmarks, IEnumerable<IFilter> filters) 
             => benchmarks.Where(benchmark => filters.All(filter => filter.Predicate(benchmark))).ToImmutableArray();
 
         private static void AssertMethodHasCorrectSignature(string methodType, MethodInfo methodInfo)

src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs

@@ -13,7 +13,7 @@ public static class BenchmarkPartitioner
     {
         public static BuildPartition[] CreateForBuild(BenchmarkRunInfo[] supportedBenchmarks, IResolver resolver)
             => supportedBenchmarks
-                .SelectMany(info => info.BenchmarksCases.Select(benchmark => (benchmark, info.Config)))
+                .SelectMany(info => info.BenchmarksCases.Select(benchmark => (benchmark, benchmark.Config)))
                 .GroupBy(tuple => tuple.benchmark, BenchmarkRuntimePropertiesComparer.Instance)
                 .Select(group => new BuildPartition(group.Select((item, index) => new BenchmarkBuildInfo(item.benchmark, item.Config, index)).ToArray(), resolver))
                 .ToArray();

src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs

@@ -29,13 +29,6 @@ public static Summary Run(Type type, IConfig config = null)
                 return RunWithDirtyAssemblyResolveHelper(type, config);
         }
 
-        [PublicAPI]
-        public static Summary Run(Type type, MethodInfo[] methods, IConfig config = null)
-        {
-            using (DirtyAssemblyResolveHelper.Create())
-                return RunWithDirtyAssemblyResolveHelper(type, methods, config);
-        }
-
         [PublicAPI]
         public static Summary[] Run(Assembly assembly, IConfig config = null)
         {
@@ -75,10 +68,6 @@ public static Summary RunSource(string source, IConfig config = null)
         private static Summary RunWithDirtyAssemblyResolveHelper(Type type, IConfig config)
             => BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(type, config) }).Single();
 
-        [MethodImpl(MethodImplOptions.NoInlining)]
-        private static Summary RunWithDirtyAssemblyResolveHelper(Type type, MethodInfo[] methods, IConfig config = null)
-            => BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.MethodsToBenchmarks(type, methods, config) }).Single();
-
         [MethodImpl(MethodImplOptions.NoInlining)]
         private static Summary[] RunWithDirtyAssemblyResolveHelper(Assembly assembly, IConfig config = null)
             => BenchmarkRunnerClean.Run(assembly.GetRunnableBenchmarks().Select(type => BenchmarkConverter.TypeToBenchmarks(type, config)).ToArray());

tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs

@@ -60,7 +60,7 @@ protected Reports.Summary CanExecute(Type type, IConfig config = null, bool full
                 config = config.With(DefaultColumnProviders.Instance);
 
             // Make sure we ALWAYS combine the Config (default or passed in) with any Config applied to the Type/Class
-            var summary = BenchmarkRunner.Run(type, BenchmarkConverter.GetFullConfig(type, config));
+            var summary = BenchmarkRunner.Run(type, config);
 
             if (fullValidation)
             {

tests/BenchmarkDotNet.Tests/Configs/ConfigPerMethodTests.cs

@@ -0,0 +1,107 @@
+using System;
+using System.Runtime.InteropServices;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Filters;
+using BenchmarkDotNet.Running;
+using Xunit;
+
+namespace BenchmarkDotNet.Tests.Configs
+{
+    public class ConfigPerMethodTests
+    {
+        [Fact]
+        public void PerMethodConfigsAreRespected()
+        {
+            var never = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkThatShouldNeverRun));
+
+            Assert.Empty(never.BenchmarksCases);
+
+            var always = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkThatShouldAlwaysRun));
+
+            Assert.NotEmpty(always.BenchmarksCases);
+        }
+
+        public class ConditionalRun : FilterConfigBaseAttribute
+        {
+            public ConditionalRun(bool value) : base(new SimpleFilter(_ => value)) { }
+        }
+
+        public class WithBenchmarkThatShouldNeverRun
+        {
+            [Benchmark]
+            [ConditionalRun(false)]
+            public void Method() { }
+        }
+
+        public class WithBenchmarkThatShouldAlwaysRun
+        {
+            [Benchmark]
+            [ConditionalRun(true)]
+            public void Method() { }
+        }
+
+        [Fact]
+        public void CanEnableOrDisableTheBenchmarkPerOperatingSystem()
+        {
+            var allowedForWindows = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkAllowedForWindows));
+            var notAllowedForWindows = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkNotAllowedForWindows));
+
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                Assert.NotEmpty(allowedForWindows.BenchmarksCases);
+                Assert.Empty(notAllowedForWindows.BenchmarksCases);
+            }
+            else
+            {
+                Assert.Empty(allowedForWindows.BenchmarksCases);
+                Assert.NotEmpty(notAllowedForWindows.BenchmarksCases);
+            }
+        }
+
+        public class WithBenchmarkAllowedForWindows
+        {
+            [Benchmark]
+            [OperatingSystemsFilter(allowed: true, PlatformID.Win32NT)]
+            public void Method() { }
+        }
+
+        public class WithBenchmarkNotAllowedForWindows
+        {
+            [Benchmark]
+            [OperatingSystemsFilter(allowed: false, PlatformID.Win32NT)]
+            public void Method() { }
+        }
+
+        [Fact]
+        public void CanEnableOrDisableTheBenchmarkPerOperatingSystemArchitecture()
+        {
+            var allowed = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkAllowedForX64));
+            var notallowed = BenchmarkConverter.TypeToBenchmarks(typeof(WithBenchmarkNotAllowedForX64));
+
+            if (RuntimeInformation.OSArchitecture == Architecture.X64)
+            {
+                Assert.NotEmpty(allowed.BenchmarksCases);
+                Assert.Empty(notallowed.BenchmarksCases);
+            }
+            else
+            {
+                Assert.Empty(allowed.BenchmarksCases);
+                Assert.NotEmpty(notallowed.BenchmarksCases);
+            }
+        }
+
+        public class WithBenchmarkAllowedForX64
+        {
+            [Benchmark]
+            [OperatingSystemsArchitectureFilter(allowed: true, Architecture.X64)]
+            public void Method() { }
+        }
+
+        public class WithBenchmarkNotAllowedForX64
+        {
+            [Benchmark]
+            [OperatingSystemsArchitectureFilter(allowed: false, Architecture.X64)]
+            public void Method() { }
+        }
+    }
+}