Improve support for NativeAOT (#1960)

* remove Obsolete fields

* add a possibility to disable specific benchmarks for AOT runtimes

* remove support for old experimental builds of CoreRT that are targeting runtimes that are not longer supported (nobody should be using this combination)

* introduce NativeAotRuntime

* rename --coreRtVersion to --ilCompilerVersion (not dependent on CoreRT/NativeAOT naming)

* update default ILCompilerVersion and NuGet feed url to point to latest 7.0 bits

* IsCoreRT should return false on .NET 7

* annotate GenericTypeArgumentsAttribute with [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)]

Author: adamsitnik

src/BenchmarkDotNet.Annotations/Attributes/DynamicallyAccessedMemberTypes.cs

@@ -0,0 +1,90 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Diagnostics.CodeAnalysis
+{
+    /// <summary>
+    /// Specifies the types of members that are dynamically accessed.
+    ///
+    /// This enumeration has a <see cref="FlagsAttribute"/> attribute that allows a
+    /// bitwise combination of its member values.
+    /// </summary>
+    [Flags]
+    internal enum DynamicallyAccessedMemberTypes
+    {
+        /// <summary>
+        /// Specifies no members.
+        /// </summary>
+        None = 0,
+
+        /// <summary>
+        /// Specifies the default, parameterless public constructor.
+        /// </summary>
+        PublicParameterlessConstructor = 0x0001,
+
+        /// <summary>
+        /// Specifies all public constructors.
+        /// </summary>
+        PublicConstructors = 0x0002 | PublicParameterlessConstructor,
+
+        /// <summary>
+        /// Specifies all non-public constructors.
+        /// </summary>
+        NonPublicConstructors = 0x0004,
+
+        /// <summary>
+        /// Specifies all public methods.
+        /// </summary>
+        PublicMethods = 0x0008,
+
+        /// <summary>
+        /// Specifies all non-public methods.
+        /// </summary>
+        NonPublicMethods = 0x0010,
+
+        /// <summary>
+        /// Specifies all public fields.
+        /// </summary>
+        PublicFields = 0x0020,
+
+        /// <summary>
+        /// Specifies all non-public fields.
+        /// </summary>
+        NonPublicFields = 0x0040,
+
+        /// <summary>
+        /// Specifies all public nested types.
+        /// </summary>
+        PublicNestedTypes = 0x0080,
+
+        /// <summary>
+        /// Specifies all non-public nested types.
+        /// </summary>
+        NonPublicNestedTypes = 0x0100,
+
+        /// <summary>
+        /// Specifies all public properties.
+        /// </summary>
+        PublicProperties = 0x0200,
+
+        /// <summary>
+        /// Specifies all non-public properties.
+        /// </summary>
+        NonPublicProperties = 0x0400,
+
+        /// <summary>
+        /// Specifies all public events.
+        /// </summary>
+        PublicEvents = 0x0800,
+
+        /// <summary>
+        /// Specifies all non-public events.
+        /// </summary>
+        NonPublicEvents = 0x1000,
+
+        /// <summary>
+        /// Specifies all members.
+        /// </summary>
+        All = ~None
+    }
+}

src/BenchmarkDotNet.Annotations/Attributes/DynamicallyAccessedMembersAttribute.cs

@@ -0,0 +1,46 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Diagnostics.CodeAnalysis
+{
+    /// <summary>
+    /// Indicates that certain members on a specified <see cref="Type"/> are accessed dynamically,
+    /// for example through <see cref="System.Reflection"/>.
+    /// </summary>
+    /// <remarks>
+    /// This allows tools to understand which members are being accessed during the execution
+    /// of a program.
+    ///
+    /// This attribute is valid on members whose type is <see cref="Type"/> or <see cref="string"/>.
+    ///
+    /// When this attribute is applied to a location of type <see cref="string"/>, the assumption is
+    /// that the string represents a fully qualified type name.
+    ///
+    /// If the attribute is applied to a method it's treated as a special case and it implies
+    /// the attribute should be applied to the "this" parameter of the method. As such the attribute
+    /// should only be used on instance methods of types assignable to System.Type (or string, but no methods
+    /// will use it there).
+    /// </remarks>
+    [AttributeUsage(
+        AttributeTargets.Field | AttributeTargets.ReturnValue | AttributeTargets.GenericParameter |
+        AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.Method,
+        Inherited = false)]
+    internal sealed class DynamicallyAccessedMembersAttribute : Attribute
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="DynamicallyAccessedMembersAttribute"/> class
+        /// with the specified member types.
+        /// </summary>
+        /// <param name="memberTypes">The types of members dynamically accessed.</param>
+        public DynamicallyAccessedMembersAttribute(DynamicallyAccessedMemberTypes memberTypes)
+        {
+            MemberTypes = memberTypes;
+        }
+
+        /// <summary>
+        /// Gets the <see cref="DynamicallyAccessedMemberTypes"/> which specifies the type
+        /// of members dynamically accessed.
+        /// </summary>
+        public DynamicallyAccessedMemberTypes MemberTypes { get; }
+    }
+}

src/BenchmarkDotNet.Annotations/Attributes/GenericTypeArgumentsAttribute.cs

@@ -1,5 +1,6 @@
 using JetBrains.Annotations;
 using System;
+using System.Diagnostics.CodeAnalysis;
 
 namespace BenchmarkDotNet.Attributes
 {
@@ -11,6 +12,6 @@ public class GenericTypeArgumentsAttribute : Attribute
         // CLS-Compliant Code requires a constructor without an array in the argument list
         [PublicAPI] public GenericTypeArgumentsAttribute() => GenericTypeArguments = new Type[0];
 
-        public GenericTypeArgumentsAttribute(params Type[] genericTypeArguments) => GenericTypeArguments = genericTypeArguments;
+        public GenericTypeArgumentsAttribute([DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)] params Type[] genericTypeArguments) => GenericTypeArguments = genericTypeArguments;
     }
 }

src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs

@@ -95,31 +95,6 @@ public enum RuntimeMoniker
         /// </summary>
         Net70,
 
-        /// <summary>
-        /// CoreRT compiled as netcoreapp2.0
-        /// </summary>
-        CoreRt20,
-
-        /// <summary>
-        /// CoreRT compiled as netcoreapp2.1
-        /// </summary>
-        CoreRt21,
-
-        /// <summary>
-        /// CoreRT compiled as netcoreapp2.2
-        /// </summary>
-        CoreRt22,
-
-        /// <summary>
-        /// CoreRT compiled as netcoreapp3.0
-        /// </summary>
-        CoreRt30,
-
-        /// <summary>
-        /// CoreRT compiled as netcoreapp3.1
-        /// </summary>
-        CoreRt31,
-
         /// <summary>
         /// CoreRT compiled as net5.0
         /// </summary>
@@ -131,9 +106,9 @@ public enum RuntimeMoniker
         CoreRt60,
 
         /// <summary>
-        /// CoreRT compiled as net7.0
+        /// NativeAOT compiled as net7.0
         /// </summary>
-        CoreRt70,
+        NativeAot70,
 
         /// <summary>
         /// WebAssembly with default .Net version

src/BenchmarkDotNet/Attributes/Filters/AotFilterAttribute.cs

@@ -0,0 +1,12 @@
+using BenchmarkDotNet.Filters;
+
+namespace BenchmarkDotNet.Attributes.Filters
+{
+    public class AotFilterAttribute : FilterConfigBaseAttribute
+    {
+        public AotFilterAttribute(string reason = null)
+            : base(new SimpleFilter(benchmark => !benchmark.GetRuntime().IsAOT))
+        {
+        }
+    }
+}

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -94,8 +94,8 @@ public class CommandLineOptions
         [Option("clrVersion", Required = false, HelpText = "Optional version of private CLR build used as the value of COMPLUS_Version env var.")]
         public string ClrVersion { get; set; }
 
-        [Option("coreRtVersion", Required = false, HelpText = "Optional version of Microsoft.DotNet.ILCompiler which should be used to run with CoreRT. Example: \"1.0.0-alpha-26414-01\"")]
-        public string CoreRtVersion { get; set; }
+        [Option("ilCompilerVersion", Required = false, HelpText = "Optional version of Microsoft.DotNet.ILCompiler which should be used to run with CoreRT/NativeAOT. Example: \"7.0.0-preview.3.22123.2\"")]
+        public string ILCompilerVersion { get; set; }
 
         [Option("ilcPath", Required = false, HelpText = "Optional IlcPath which should be used to run with private CoreRT build.")]
         public DirectoryInfo CoreRtPath { get; set; }

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -363,32 +363,11 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma
                         .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName)));
                 case RuntimeMoniker.Mono:
                     return baseJob.WithRuntime(new MonoRuntime("Mono", options.MonoPath?.FullName));
-                case RuntimeMoniker.CoreRt20:
-                case RuntimeMoniker.CoreRt21:
-                case RuntimeMoniker.CoreRt22:
-                case RuntimeMoniker.CoreRt30:
-                case RuntimeMoniker.CoreRt31:
                 case RuntimeMoniker.CoreRt50:
                 case RuntimeMoniker.CoreRt60:
-                case RuntimeMoniker.CoreRt70:
-                    var builder = CoreRtToolchain.CreateBuilder();
-
-                    if (options.CliPath != null)
-                        builder.DotNetCli(options.CliPath.FullName);
-                    if (options.RestorePath != null)
-                        builder.PackagesRestorePath(options.RestorePath.FullName);
-
-                    if (options.CoreRtPath != null)
-                        builder.UseCoreRtLocal(options.CoreRtPath.FullName);
-                    else if (!string.IsNullOrEmpty(options.CoreRtVersion))
-                        builder.UseCoreRtNuGet(options.CoreRtVersion);
-                    else
-                        builder.UseCoreRtNuGet();
-
-                    var runtime = runtimeMoniker.GetRuntime();
-                    builder.TargetFrameworkMoniker(runtime.MsBuildMoniker);
-
-                    return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain());
+                    return CreateAotJob(baseJob, options, runtimeMoniker, "6.0.0-*", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json");
+                case RuntimeMoniker.NativeAot70:
+                    return CreateAotJob(baseJob, options, runtimeMoniker, "7.0.0-*", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json");
                 case RuntimeMoniker.Wasm:
                     return MakeWasmJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net5.0", runtimeMoniker);
                 case RuntimeMoniker.WasmNet50:
@@ -408,6 +387,28 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma
             }
         }
 
+        private static Job CreateAotJob(Job baseJob, CommandLineOptions options, RuntimeMoniker runtimeMoniker, string ilCompilerVersion, string nuGetFeedUrl)
+        {
+            var builder = CoreRtToolchain.CreateBuilder();
+
+            if (options.CliPath != null)
+                builder.DotNetCli(options.CliPath.FullName);
+            if (options.RestorePath != null)
+                builder.PackagesRestorePath(options.RestorePath.FullName);
+
+            if (options.CoreRtPath != null)
+                builder.UseCoreRtLocal(options.CoreRtPath.FullName);
+            else if (!string.IsNullOrEmpty(options.ILCompilerVersion))
+                builder.UseCoreRtNuGet(options.ILCompilerVersion, nuGetFeedUrl);
+            else
+                builder.UseCoreRtNuGet(ilCompilerVersion, nuGetFeedUrl);
+
+            var runtime = runtimeMoniker.GetRuntime();
+            builder.TargetFrameworkMoniker(runtime.MsBuildMoniker);
+
+            return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain());
+        }
+
         private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, string msBuildMoniker)
         {
             var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, msBuildMoniker: msBuildMoniker);

src/BenchmarkDotNet/Environments/Runtimes/CoreRtRuntime.cs

@@ -1,32 +1,11 @@
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Portability;
 using System;
-using System.Linq;
 
 namespace BenchmarkDotNet.Environments
 {
     public class CoreRtRuntime : Runtime
     {
-        /// <summary>
-        /// CoreRT compiled as netcoreapp2.0
-        /// </summary>
-        public static readonly CoreRtRuntime CoreRt20 = new CoreRtRuntime(RuntimeMoniker.CoreRt20, "netcoreapp2.0", "CoreRT 2.0");
-        /// <summary>
-        /// CoreRT compiled as netcoreapp2.1
-        /// </summary>
-        public static readonly CoreRtRuntime CoreRt21 = new CoreRtRuntime(RuntimeMoniker.CoreRt21, "netcoreapp2.1", "CoreRT 2.1");
-        /// <summary>
-        /// CoreRT compiled as netcoreapp2.2
-        /// </summary>
-        public static readonly CoreRtRuntime CoreRt22 = new CoreRtRuntime(RuntimeMoniker.CoreRt22, "netcoreapp2.2", "CoreRT 2.2");
-        /// <summary>
-        /// CoreRT compiled as netcoreapp3.0
-        /// </summary>
-        public static readonly CoreRtRuntime CoreRt30 = new CoreRtRuntime(RuntimeMoniker.CoreRt30, "netcoreapp3.0", "CoreRT 3.0");
-        /// <summary>
-        /// CoreRT compiled as netcoreapp3.1
-        /// </summary>
-        public static readonly CoreRtRuntime CoreRt31 = new CoreRtRuntime(RuntimeMoniker.CoreRt31, "netcoreapp3.1", "CoreRT 3.1");
         /// <summary>
         /// CoreRT compiled as net5.0
         /// </summary>
@@ -35,10 +14,8 @@ public class CoreRtRuntime : Runtime
         /// CoreRT compiled as net6.0
         /// </summary>
         public static readonly CoreRtRuntime CoreRt60 = new CoreRtRuntime(RuntimeMoniker.CoreRt60, "net6.0", "CoreRT 6.0");
-        /// <summary>
-        /// CoreRT compiled as net7.0
-        /// </summary>
-        public static readonly CoreRtRuntime CoreRt70 = new CoreRtRuntime(RuntimeMoniker.CoreRt70, "net7.0", "CoreRT 7.0");
+
+        public override bool IsAOT => true;
 
         private CoreRtRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName)
             : base(runtimeMoniker, msBuildMoniker, displayName)
@@ -59,14 +36,8 @@ public static CoreRtRuntime GetCurrentVersion()
 
             switch (version)
             {
-                case Version v when v.Major == 2 && v.Minor == 0: return CoreRt20;
-                case Version v when v.Major == 2 && v.Minor == 1: return CoreRt21;
-                case Version v when v.Major == 2 && v.Minor == 2: return CoreRt22;
-                case Version v when v.Major == 3 && v.Minor == 0: return CoreRt30;
-                case Version v when v.Major == 3 && v.Minor == 1: return CoreRt31;
                 case Version v when v.Major == 5 && v.Minor == 0: return CoreRt50;
                 case Version v when v.Major == 6 && v.Minor == 0: return CoreRt60;
-                case Version v when v.Major == 7 && v.Minor == 0: return CoreRt70;
                 default:
                     return new CoreRtRuntime(RuntimeMoniker.NotRecognized, $"net{version.Major}.{version.Minor}", $"CoreRT {version.Major}.{version.Minor}");
             }

src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs

@@ -13,6 +13,8 @@ public class MonoAotLLVMRuntime : Runtime, IEquatable<MonoAotLLVMRuntime>
 
         public FileInfo AOTCompilerPath { get; }
 
+        public override bool IsAOT => true;
+
         /// <summary>
         /// creates new instance of MonoAotLLVMRuntime
         /// </summary>

src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs

@@ -11,6 +11,8 @@ public class MonoRuntime : Runtime, IEquatable<MonoRuntime>
 
         public string AotArgs { get; }
 
+        public override bool IsAOT => !string.IsNullOrEmpty(AotArgs);
+
         public string MonoBclPath { get; }
 
         private MonoRuntime(string name) : base(RuntimeMoniker.Mono, "mono", name) { }