Add support for Platform-specific TFMs introduced in .NET 5 (#1560)

* new versions of .NET Core should not use "netcoreapp" word in the target framework moniker

example: .NET Core 6.0 is just .NET 6 and has a "net6.0" moniker

* add tests that ensure that new Platform-specific TFMs are recognized by BenchmarkPartitioner

* recognize Platform-specific apps using... reflection ;)

* make sure that a platform-specific TFM creates corresponding toolchain that is equatable with other instances

* extend console args support with the platform-specific monikers

Author: adamsitnik

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -106,12 +106,12 @@ private static bool Validate(CommandLineOptions options, ILogger logger)
 
             foreach (string runtime in options.Runtimes)
             {
-                if (!Enum.TryParse<RuntimeMoniker>(runtime.Replace(".", string.Empty), ignoreCase: true, out var parsed))
+                if (!TryParse(runtime, out RuntimeMoniker runtimeMoniker))
                 {
                     logger.WriteLineError($"The provided runtime \"{runtime}\" is invalid. Available options are: {string.Join(", ", Enum.GetNames(typeof(RuntimeMoniker)).Select(name => name.ToLower()))}.");
                     return false;
                 }
-                else if (parsed == RuntimeMoniker.Wasm && (options.WasmMainJs == null || options.WasmMainJs.IsNotNullButDoesNotExist()))
+                else if (runtimeMoniker == RuntimeMoniker.Wasm && (options.WasmMainJs == null || options.WasmMainJs.IsNotNullButDoesNotExist()))
                 {
                     logger.WriteLineError($"The provided {nameof(options.WasmMainJs)} \"{options.WasmMainJs}\" does NOT exist. It MUST be provided.");
                     return false;
@@ -319,7 +319,7 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma
         {
             TimeSpan? timeOut = options.TimeOutInSeconds.HasValue ? TimeSpan.FromSeconds(options.TimeOutInSeconds.Value) : default(TimeSpan?);
 
-            if (!Enum.TryParse(runtimeId.Replace(".", string.Empty), ignoreCase: true, out RuntimeMoniker runtimeMoniker))
+            if (!TryParse(runtimeId, out RuntimeMoniker runtimeMoniker))
             {
                 throw new InvalidOperationException("Impossible, already validated by the Validate method");
             }
@@ -481,5 +481,14 @@ private static string GetCoreRunToolchainDisplayName(IReadOnlyList<FileInfo> pat
 
             return coreRunPath.FullName.Substring(lastCommonDirectorySeparatorIndex);
         }
+
+        private static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker)
+        {
+            int index = runtime.IndexOf('-');
+
+            return index < 0
+                ? Enum.TryParse<RuntimeMoniker>(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker)
+                : Enum.TryParse<RuntimeMoniker>(runtime.Substring(0, index).Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker);
+        }
     }
 }

src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs

@@ -24,6 +24,8 @@ private CoreRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string
         {
         }
 
+        public bool IsPlatformSpecific => MsBuildMoniker.IndexOf('-') > 0;
+
         /// <summary>
         /// use this method if you want to target .NET Core version not supported by current version of BenchmarkDotNet. Example: .NET Core 10
         /// </summary>
@@ -62,9 +64,10 @@ internal static CoreRuntime FromVersion(Version version)
                 case Version v when v.Major == 2 && v.Minor == 2: return Core22;
                 case Version v when v.Major == 3 && v.Minor == 0: return Core30;
                 case Version v when v.Major == 3 && v.Minor == 1: return Core31;
-                case Version v when v.Major == 5 && v.Minor == 0: return Core50;
+                case Version v when v.Major == 5 && v.Minor == 0: return GetPlatformSpecific(Core50);
+                case Version v when v.Major == 6 && v.Minor == 0: return GetPlatformSpecific(Core60);
                 default:
-                    return CreateForNewVersion($"netcoreapp{version.Major}.{version.Minor}", $".NET Core {version.Major}.{version.Minor}");
+                    return CreateForNewVersion($"net{version.Major}.{version.Minor}", $".NET {version.Major}.{version.Minor}");
             }
         }
 
@@ -172,5 +175,32 @@ internal static bool TryGetVersionFromFrameworkName(string frameworkName, out Ve
 
         // Version.TryParse does not handle thing like 3.0.0-WORD
         private static string GetParsableVersionPart(string fullVersionName) => new string(fullVersionName.TakeWhile(c => char.IsDigit(c) || c == '.').ToArray());
+
+        private static CoreRuntime GetPlatformSpecific(CoreRuntime fallback)
+        {
+            // TargetPlatformAttribute is not part of .NET Standard 2.0 so as usuall we have to use some reflection hacks...
+            var targetPlatformAttributeType = typeof(object).Assembly.GetType("System.Runtime.Versioning.TargetPlatformAttribute", throwOnError: false);
+            if (targetPlatformAttributeType is null) // an old preview version of .NET 5
+                return fallback;
+
+            var exe = Assembly.GetEntryAssembly();
+            if (exe is null)
+                return fallback;
+
+            var attributeInstance = exe.GetCustomAttribute(targetPlatformAttributeType);
+            if (attributeInstance is null)
+                return fallback;
+
+            var platformNameProperty = targetPlatformAttributeType.GetProperty("PlatformName");
+            if (platformNameProperty is null)
+                return fallback;
+
+            if (!(platformNameProperty.GetValue(attributeInstance) is string platformName))
+                return fallback;
+
+            // it's something like "Windows7.0";
+            var justName = new string(platformName.TakeWhile(char.IsLetter).ToArray());
+            return new CoreRuntime(fallback.RuntimeMoniker, $"{fallback.MsBuildMoniker}-{justName}", fallback.Name);
+        }
     }
 }

src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs

@@ -7,11 +7,12 @@
 using BenchmarkDotNet.Toolchains.DotNetCli;
 using BenchmarkDotNet.Toolchains.InProcess.Emit;
 using JetBrains.Annotations;
+using System;
 
 namespace BenchmarkDotNet.Toolchains.CsProj
 {
     [PublicAPI]
-    public class CsProjCoreToolchain : Toolchain
+    public class CsProjCoreToolchain : Toolchain, IEquatable<CsProjCoreToolchain>
     {
         [PublicAPI] public static readonly IToolchain NetCoreApp20 = From(NetCoreAppSettings.NetCoreApp20);
         [PublicAPI] public static readonly IToolchain NetCoreApp21 = From(NetCoreAppSettings.NetCoreApp21);
@@ -70,5 +71,11 @@ public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IR
 
             return true;
         }
+
+        public override bool Equals(object obj) => obj is CsProjCoreToolchain typed && Equals(typed);
+
+        public bool Equals(CsProjCoreToolchain other) => Generator.Equals(other.Generator);
+
+        public override int GetHashCode() => Generator.GetHashCode();
     }
 }

src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs

@@ -18,7 +18,7 @@
 namespace BenchmarkDotNet.Toolchains.CsProj
 {
     [PublicAPI]
-    public class CsProjGenerator : DotNetCliGenerator
+    public class CsProjGenerator : DotNetCliGenerator, IEquatable<CsProjGenerator>
     {
         private const string DefaultSdkName = "Microsoft.NET.Sdk";
 
@@ -169,5 +169,19 @@ protected virtual FileInfo GetProjectFilePath(Type benchmarkTarget, ILogger logg
             }
             return projectFile;
         }
+
+        public override bool Equals(object obj) => obj is CsProjGenerator other && Equals(other);
+
+        public bool Equals(CsProjGenerator other)
+            => TargetFrameworkMoniker == other.TargetFrameworkMoniker
+                && RuntimeFrameworkVersion == other.RuntimeFrameworkVersion
+                && CliPath == other.CliPath
+                && PackagesPath == other.PackagesPath;
+
+        public override int GetHashCode()
+            => TargetFrameworkMoniker.GetHashCode()
+                ^ (RuntimeFrameworkVersion?.GetHashCode() ?? 0)
+                ^ (CliPath?.GetHashCode() ?? 0)
+                ^ (PackagesPath?.GetHashCode() ?? 0);
     }
 }

src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs

@@ -55,7 +55,7 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor descrip
                 case CoreRuntime coreRuntime:
                     if (descriptor != null && descriptor.Type.Assembly.IsLinqPad())
                         return InProcessEmitToolchain.Instance;
-                    if (coreRuntime.RuntimeMoniker != RuntimeMoniker.NotRecognized)
+                    if (coreRuntime.RuntimeMoniker != RuntimeMoniker.NotRecognized && !coreRuntime.IsPlatformSpecific)
                         return GetToolchain(coreRuntime.RuntimeMoniker);
 
                     return CsProjCoreToolchain.From(new NetCoreAppSettings(coreRuntime.MsBuildMoniker, null, coreRuntime.Name));

tests/BenchmarkDotNet.Tests/ConfigParserTests.cs

@@ -319,6 +319,19 @@ public void Net50AndNet60MonikersAreRecognizedAsNetCoreMonikers(string tfm)
             Assert.Equal(tfm, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker);
         }
 
+        [Theory]
+        [InlineData("net5.0-windows")]
+        [InlineData("net5.0-ios")]
+        public void PlatformSpecificMonikersAreSupported(string msBuildMoniker)
+        {
+            var config = ConfigParser.Parse(new[] { "-r", msBuildMoniker }, new OutputLogger(Output)).config;
+
+            Assert.Single(config.GetJobs());
+            CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain;
+            Assert.NotNull(toolchain);
+            Assert.Equal(msBuildMoniker, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker);
+        }
+
         [Fact]
         public void CanCompareFewDifferentRuntimes()
         {

tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs

@@ -5,6 +5,7 @@
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Tests.XUnit;
+using BenchmarkDotNet.Toolchains.CsProj;
 using Xunit;
 
 namespace BenchmarkDotNet.Tests.Running
@@ -39,6 +40,13 @@ [Benchmark] public void M2() { }
             [Benchmark] public void M3() { }
         }
 
+        public class Plain3
+        {
+            [Benchmark] public void M1() { }
+            [Benchmark] public void M2() { }
+            [Benchmark] public void M3() { }
+        }
+
         [Fact]
         public void BenchmarksAreGroupedByJob()
         {
@@ -128,5 +136,38 @@ public void CustomNuGetJobsAreGroupedByPackageVersion()
             foreach (var grouping in grouped)
                 Assert.Equal(3 * 2, grouping.Count()); // (M1 + M2 + M3) * (Plain1 + Plain2)
         }
+
+        [Fact]
+        public void CustomTargetPlatformJobsAreGroupedByTargetFrameworkMoniker()
+        {
+            var net5Config = ManualConfig.Create(DefaultConfig.Instance)
+                .AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp50));
+            var net5WindowsConfig1 = ManualConfig.Create(DefaultConfig.Instance)
+                .AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.From(new Toolchains.DotNetCli.NetCoreAppSettings(
+                    targetFrameworkMoniker: "net5.0-windows",
+                    runtimeFrameworkVersion: null,
+                    name: ".NET 5.0"))));
+            // a different INSTANCE of CsProjCoreToolchain that also targets "net5.0-windows"
+            var net5WindowsConfig2 = ManualConfig.Create(DefaultConfig.Instance)
+                .AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.From(new Toolchains.DotNetCli.NetCoreAppSettings(
+                    targetFrameworkMoniker: "net5.0-windows",
+                    runtimeFrameworkVersion: null,
+                    name: ".NET 5.0"))));
+
+            var benchmarksNet5 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain1), net5Config);
+            var benchmarksNet5Windows1 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain2), net5WindowsConfig1);
+            var benchmarksNet5Windows2 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain3), net5WindowsConfig2);
+
+            var grouped = benchmarksNet5.BenchmarksCases
+                .Union(benchmarksNet5Windows1.BenchmarksCases)
+                .Union(benchmarksNet5Windows2.BenchmarksCases)
+                .GroupBy(benchmark => benchmark, new BenchmarkPartitioner.BenchmarkRuntimePropertiesComparer())
+                .ToArray();
+
+            Assert.Equal(2, grouped.Length);
+
+            Assert.Single(grouped, group => group.Count() == 3); // Plain1 (3 methods) runing against "net5.0"
+            Assert.Single(grouped, group => group.Count() == 6); // Plain2 (3 methods) and Plain3 (3 methods) runing against "net5.0-windows"
+        }
     }
 }

tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs

@@ -17,7 +17,7 @@ public class RuntimeVersionDetectionTests
         [InlineData(".NETCoreApp,Version=v3.0", RuntimeMoniker.NetCoreApp30, "netcoreapp3.0")]
         [InlineData(".NETCoreApp,Version=v3.1", RuntimeMoniker.NetCoreApp31, "netcoreapp3.1")]
         [InlineData(".NETCoreApp,Version=v5.0", RuntimeMoniker.Net50, "net5.0")]
-        [InlineData(".NETCoreApp,Version=v123.0", RuntimeMoniker.NotRecognized, "netcoreapp123.0")]
+        [InlineData(".NETCoreApp,Version=v123.0", RuntimeMoniker.NotRecognized, "net123.0")]
         public void TryGetVersionFromFrameworkNameHandlesValidInput(string frameworkName, RuntimeMoniker expectedTfm, string expectedMsBuildMoniker)
         {
             Assert.True(CoreRuntime.TryGetVersionFromFrameworkName(frameworkName, out Version version));
@@ -44,7 +44,7 @@ public void TryGetVersionFromFrameworkNameHandlesInvalidInput(string frameworkNa
         [InlineData(RuntimeMoniker.NetCoreApp30, "netcoreapp3.0", "Microsoft .NET Core", "3.0.0-preview8-28379-12")]
         [InlineData(RuntimeMoniker.NetCoreApp31, "netcoreapp3.1", "Microsoft .NET Core", "3.1.0-something")]
         [InlineData(RuntimeMoniker.Net50, "net5.0", "Microsoft .NET Core", "5.0.0-alpha1.19415.3")]
-        [InlineData(RuntimeMoniker.NotRecognized, "netcoreapp123.0", "Microsoft .NET Core", "123.0.0-future")]
+        [InlineData(RuntimeMoniker.NotRecognized, "net123.0", "Microsoft .NET Core", "123.0.0-future")]
         public void TryGetVersionFromProductInfoHandlesValidInput(RuntimeMoniker expectedTfm, string expectedMsBuildMoniker, string productName, string productVersion)
         {
             Assert.True(CoreRuntime.TryGetVersionFromProductInfo(productVersion, productName, out Version version));
@@ -74,7 +74,7 @@ public static IEnumerable<object[]> FromNetCoreAppVersionHandlesValidInputArgume
             yield return new object[] { Path.Combine(directoryPrefix, "2.2.6") + Path.DirectorySeparatorChar, RuntimeMoniker.NetCoreApp22, "netcoreapp2.2" };
             yield return new object[] { Path.Combine(directoryPrefix, "3.0.0-preview8-28379-12") + Path.DirectorySeparatorChar, RuntimeMoniker.NetCoreApp30, "netcoreapp3.0" };
             yield return new object[] { Path.Combine(directoryPrefix, "5.0.0-alpha1.19422.13") + Path.DirectorySeparatorChar, RuntimeMoniker.Net50, "net5.0" };
-            yield return new object[] { Path.Combine(directoryPrefix, "123.0.0") + Path.DirectorySeparatorChar, RuntimeMoniker.NotRecognized, "netcoreapp123.0" };
+            yield return new object[] { Path.Combine(directoryPrefix, "123.0.0") + Path.DirectorySeparatorChar, RuntimeMoniker.NotRecognized, "net123.0" };
         }
 
         [Theory]