Add Wasm Tool Chain (#1483)

Author: Nathan Ricci

samples/BenchmarkDotNet.Samples/nuget.config

@@ -0,0 +1,5 @@
+<configuration>
+  <packageSources>
+    <add key="dotnet5" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json" />
+  </packageSources>
+</configuration>

src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs

@@ -105,6 +105,11 @@ public enum RuntimeMoniker
         /// <summary>
         /// CoreRT compiled as netcoreapp5.0
         /// </summary>
-        CoreRt50
+        CoreRt50,
+
+        /// <summary>
+        /// WebAssembly
+        /// </summary>
+        Wasm
     }
-}
+}

src/BenchmarkDotNet/BenchmarkDotNet.csproj

@@ -49,4 +49,4 @@
     <Compile Include="..\BenchmarkDotNet.Disassembler.x64\SourceCodeProvider.cs" Link="Disassemblers\SourceCodeProvider.cs" />
     <Compile Include="..\BenchmarkDotNet.Disassembler.x64\ClrMdDisassembler.cs" Link="Disassemblers\ClrMdDisassembler.cs" />
   </ItemGroup>
-</Project>
+</Project>

src/BenchmarkDotNet/Code/CodeGenerator.cs

@@ -75,6 +75,8 @@ internal static string Generate(BuildPartition buildPartition)
                 extraDefines.Add("#define CORERT");
             else if (buildPartition.IsNetFramework)
                 extraDefines.Add("#define NETFRAMEWORK");
+            else if (buildPartition.IsWasm)
+                extraDefines.Add("#define WASM");
 
             string benchmarkProgramContent = new SmartStringBuilder(ResourceHelper.LoadTemplate("BenchmarkProgram.txt"))
                 .Replace("$ShadowCopyDefines$", useShadowCopy ? "#define SHADOWCOPY" : null).Replace("$ShadowCopyFolderPath$", shadowCopyFolderPath)

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -164,6 +164,15 @@ public class CommandLineOptions
         [Option("envVars", Required = false, HelpText = "Colon separated environment variables (key:value)")]
         public IEnumerable<string> EnvironmentVariables { get; set; }
 
+        [Option("wasmJavascriptEnginePath", Required = false, Default = "v8", HelpText = "Full path to a java script engine used to run the benchmarks, if using the WASM runtime.")]
+        public string WasmJavascriptEnginePath { get; set; }
+
+        [Option("wasmMainJS", Required = false, HelpText = "Path to the main.js file used for wasm apps.")]
+        public string WasmMainJS { get; set; }
+
+        [Option("wasmJavaScriptEngineArguments", Required = false, Default = "--expose_wasm", HelpText = "Arguments for the javascript engine used by wasm.")]
+        public string WasmJavaScriptEngineArguments { get; set; }
+
         internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any();
 
         [Usage(ApplicationAlias = "")]

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -17,11 +17,13 @@
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.CoreRt;
 using BenchmarkDotNet.Toolchains.CoreRun;
 using BenchmarkDotNet.Toolchains.CsProj;
 using BenchmarkDotNet.Toolchains.DotNetCli;
 using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using BenchmarkDotNet.Toolchains.MonoWasm;
 using CommandLine;
 using Perfolizer.Horology;
 using Perfolizer.Mathematics.OutlierDetection;
@@ -356,6 +358,21 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma
                     builder.TargetFrameworkMoniker(runtime.MsBuildMoniker);
 
                     return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain());
+                case RuntimeMoniker.Wasm:
+                    var wasmRuntime = runtimeMoniker.GetRuntime();
+
+                    WasmSettings wasmSettings = new WasmSettings(wasmMainJS: options.WasmMainJS,
+                                                                 wasmJavaScriptEngine: options.WasmJavascriptEnginePath,
+                                                                 wasmjavaScriptEngineArguments: options.WasmJavaScriptEngineArguments);
+
+                    IToolchain toolChain = new WasmToolChain(name: "Wasm",
+                                                             targetFrameworkMoniker: wasmRuntime.MsBuildMoniker,
+                                                             cliPath: options.CliPath.FullName,
+                                                             packagesPath: options.RestorePath?.FullName,
+                                                             wasmSettings: wasmSettings,
+                                                             timeout: timeOut ?? NetCoreAppSettings.DefaultBuildTimeout);
+
+                        return baseJob.WithRuntime(wasmRuntime).WithToolchain(toolChain);
                 default:
                     throw new NotSupportedException($"Runtime {runtimeId} is not supported");
             }

src/BenchmarkDotNet/Engines/NoAcknowledgementConsoleHost.cs

@@ -0,0 +1,31 @@
+using System;
+using System.IO;
+using BenchmarkDotNet.Validators;
+using JetBrains.Annotations;
+
+namespace BenchmarkDotNet.Engines
+{
+    /* This class is used by wasm because wasm cannot read from the console.
+     * This potentially breaks communication with Diagnosers. */
+    public sealed class NoAcknowledgementConsoleHost : IHost
+    {
+        private readonly TextWriter outWriter;
+
+        public NoAcknowledgementConsoleHost([NotNull]TextWriter outWriter)
+        {
+            this.outWriter = outWriter ?? throw new ArgumentNullException(nameof(outWriter));
+        }
+
+        public void Write(string message) => outWriter.Write(message);
+
+        public void WriteLine() => outWriter.WriteLine();
+
+        public void WriteLine(string message) => outWriter.WriteLine(message);
+
+        public void SendSignal(HostSignal hostSignal) => WriteLine(Engine.Signals.ToMessage(hostSignal));
+
+        public void SendError(string message) => outWriter.WriteLine($"{ValidationErrorReporter.ConsoleErrorPrefix} {message}");
+
+        public void ReportResults(RunResults runResults) => runResults.Print(outWriter);
+    }
+}

src/BenchmarkDotNet/Environments/Platform.cs

@@ -25,6 +25,11 @@ public enum Platform
         /// <summary>
         /// ARM64
         /// </summary>
-        Arm64
+        Arm64,
+
+        /// <summary>
+        /// Wasm
+        /// </summary>
+        Wasm
     }
 }

src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs

@@ -0,0 +1,22 @@
+using System;
+using BenchmarkDotNet.Jobs;
+
+namespace BenchmarkDotNet.Environments
+{
+    public class WasmRuntime : Runtime, IEquatable<WasmRuntime>
+    {
+        public static readonly WasmRuntime Default = new WasmRuntime("Wasm");
+
+        public WasmRuntime(string name) : base(RuntimeMoniker.Wasm, "net5.0", name)
+        {
+        }
+        
+        public WasmRuntime(string name, string msBuildMoniker) : base(RuntimeMoniker.Wasm, msBuildMoniker, name)
+        {
+        }
+
+        public override bool Equals(object obj) => obj is WasmRuntime other && Equals(other);
+
+        public bool Equals(WasmRuntime other) => base.Equals(other);
+    }
+}

src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs

@@ -48,6 +48,8 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker)
                     return CoreRtRuntime.CoreRt31;
                 case RuntimeMoniker.CoreRt50:
                     return CoreRtRuntime.CoreRt50;
+                case RuntimeMoniker.Wasm:
+                    return WasmRuntime.Default;
                 default:
                     throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "Runtime Moniker not supported");
             }

src/BenchmarkDotNet/Helpers/ConsoleExitHandler.cs

@@ -26,7 +26,7 @@ private void Attach()
             try
             {
                 Console.CancelKeyPress += CancelKeyPressHandlerCallback;
-            } 
+            }
             catch (PlatformNotSupportedException)
             {
                 // Thrown when running in Xamarin

src/BenchmarkDotNet/Portability/RuntimeInformation.cs

@@ -42,6 +42,11 @@ public static bool IsCoreRT
             => ((Environment.Version.Major >= 5) || FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase))
                && string.IsNullOrEmpty(typeof(object).Assembly.Location); // but it's merged to a single .exe and .Location returns null here ;)
 
+        /// <summary>
+        /// "Is the target where we will run the benchmarks WASM?"
+        /// </summary>
+        public static bool IsWasm => IsOSPlatform(OSPlatform.Create("BROWSER"));
+
         public static bool IsRunningInContainer => string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true");
 
         internal static string ExecutableExtension => IsWindows() ? ".exe" : string.Empty;
@@ -144,6 +149,10 @@ internal static string GetRuntimeVersion()
             {
                 return FrameworkVersionHelper.GetFrameworkDescription();
             }
+            else if (IsWasm)
+            {
+                return "wasm";
+            }
             else if (IsNetCore)
             {
                 string runtimeVersion = CoreRuntime.TryGetVersion(out var version) ? version.ToString() : "?";
@@ -168,6 +177,8 @@ internal static Runtime GetCurrentRuntime()
                 return MonoRuntime.Default;
             if (IsFullFramework)
                 return ClrRuntime.GetCurrentVersion();
+            if (IsWasm)
+                return WasmRuntime.Default;
             if (IsNetCore)
                 return CoreRuntime.GetCurrentVersion();
             if (IsCoreRT)
@@ -189,6 +200,8 @@ public static Platform GetCurrentPlatform()
                 case Architecture.X86:
                     return Platform.X86;
                 default:
+                    if (IsWasm)
+                        return Platform.Wasm;
                     throw new ArgumentOutOfRangeException();
             }
         }

src/BenchmarkDotNet/Running/BuildPartition.cs

@@ -6,6 +6,7 @@
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Toolchains.CoreRt;
 using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Toolchains.MonoWasm;
 using BenchmarkDotNet.Toolchains.Roslyn;
 using JetBrains.Annotations;
 
@@ -46,6 +47,9 @@ public BuildPartition(BenchmarkBuildInfo[] benchmarks, IResolver resolver)
             // given job can have CoreRT toolchain set, but Runtime == default ;)
             || (RepresentativeBenchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && toolchain is CoreRtToolchain);
 
+        public bool IsWasm => Runtime is WasmRuntime // given job can have Wasm toolchain set, but Runtime == default ;)
+            || (RepresentativeBenchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && toolchain is WasmToolChain);
+
         public bool IsNetFramework => Runtime is ClrRuntime
             || (RepresentativeBenchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && (toolchain is RoslynToolchain || toolchain is CsProjClassicNetToolchain));
 

src/BenchmarkDotNet/Templates/BenchmarkProgram.txt

@@ -24,7 +24,11 @@ namespace BenchmarkDotNet.Autogenerated
 
         private static System.Int32 AfterAssemblyLoadingAttached(System.String[] args)
         {
+#if WASM
+            BenchmarkDotNet.Engines.NoAcknowledgementConsoleHost host = new BenchmarkDotNet.Engines.NoAcknowledgementConsoleHost(System.Console.Out); // this variable name is used by CodeGenerator.GetCoreRtSwitch, do NOT change it
+#else
             BenchmarkDotNet.Engines.ConsoleHost host = new BenchmarkDotNet.Engines.ConsoleHost(System.Console.Out, System.Console.In); // this variable name is used by CodeGenerator.GetCoreRtSwitch, do NOT change it
+#endif
 
             // the first thing to do is to let diagnosers hook in before anything happens
             // so all jit-related diagnosers can catch first jit compilation!

src/BenchmarkDotNet/Templates/WasmCsProj.txt

@@ -0,0 +1,20 @@
+<Project Sdk="$SDKNAME$" DefaultTargets="Publish">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <OutputPath>bin</OutputPath>
+    <EnableDefaultCompileItems>false</EnableDefaultCompileItems>
+    <TargetFramework>$TFM$</TargetFramework>
+    <AppDir>$(MSBuildThisFileDirectory)\bin\$TFM$\browser-wasm\publish</AppDir>
+    <AssemblyName>$PROGRAMNAME$</AssemblyName>
+    <RuntimeIdentifier>browser-wasm</RuntimeIdentifier>
+    $COPIEDSETTINGS$
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="$CODEFILENAME$" Exclude="bin\**;obj\**;**\*.xproj;packages\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="$CSPROJPATH$" />
+  </ItemGroup>
+</Project>

src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs

@@ -24,7 +24,7 @@ public DotNetCliBuilder(string targetFrameworkMoniker, string customDotNetCliPat
             Timeout = timeout ?? NetCoreAppSettings.DefaultBuildTimeout;
         }
 
-        public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger)
+        public virtual BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger)
             => new DotNetCliCommand(
                     CustomDotNetCliPath,
                     string.Empty,

src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs

@@ -43,11 +43,13 @@ public NetCoreAppSettings(
             string name,
             string customDotNetCliPath = null,
             string packagesPath = null,
-            TimeSpan? timeout = null)
+            TimeSpan? timeout = null
+            )
         {
             TargetFrameworkMoniker = targetFrameworkMoniker;
             RuntimeFrameworkVersion = runtimeFrameworkVersion;
             Name = name;
+
             CustomDotNetCliPath = customDotNetCliPath;
             PackagesPath = packagesPath;
             Timeout = timeout ?? DefaultBuildTimeout;

src/BenchmarkDotNet/Toolchains/GeneratorBase.cs

@@ -111,6 +111,8 @@ [PublicAPI] protected virtual void GenerateAppConfig(BuildPartition buildPartiti
         [PublicAPI] protected virtual void GenerateCode(BuildPartition buildPartition, ArtifactsPaths artifactsPaths)
             => File.WriteAllText(artifactsPaths.ProgramCodePath, CodeGenerator.Generate(buildPartition));
 
+        protected virtual string GetExecutablePath(string binariesDirectoryPath, string programName) => Path.Combine(binariesDirectoryPath, $"{programName}{GetExecutableExtension()}");
+
         private ArtifactsPaths GetArtifactsPaths(BuildPartition buildPartition, string rootArtifactsFolderPath)
         {
             // its not ".cs" in order to avoid VS from displaying and compiling it with xprojs/csprojs that include all *.cs by default
@@ -119,7 +121,8 @@ private ArtifactsPaths GetArtifactsPaths(BuildPartition buildPartition, string r
             string programName = buildPartition.ProgramName;
             string buildArtifactsDirectoryPath = GetBuildArtifactsDirectoryPath(buildPartition, programName);
             string binariesDirectoryPath = GetBinariesDirectoryPath(buildArtifactsDirectoryPath, buildPartition.BuildConfiguration);
-            string executablePath = Path.Combine(binariesDirectoryPath, $"{programName}{GetExecutableExtension()}");
+
+            string executablePath = GetExecutablePath(binariesDirectoryPath, programName);
 
             return new ArtifactsPaths(
                 rootArtifactsFolderPath: rootArtifactsFolderPath,

src/BenchmarkDotNet/Toolchains/MonoWasm/WasmAppBuilder.cs

@@ -0,0 +1,73 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Reflection;
+using BenchmarkDotNet.Toolchains.MonoWasm;
+
+public class WasmAppBuilder
+{
+    private readonly WasmSettings WasmSettings;
+    private readonly string TargetFrameworkMoniker;
+
+    public WasmAppBuilder(WasmSettings wasmSettings, string targetFrameworkMoniker)
+    {
+        WasmSettings = wasmSettings;
+        TargetFrameworkMoniker = targetFrameworkMoniker;
+    }
+
+    public bool BuildApp (string programName, string projectRoot)
+    {
+        string[] assemblies;
+        string appDir = Path.Combine(projectRoot, $"bin", TargetFrameworkMoniker, "browser-wasm", "publish");
+        string outputDir = Path.Combine(appDir, "output");
+
+        string mainAssemblyPath = Path.Combine(appDir, $"{programName}.dll");
+
+        if (!File.Exists(mainAssemblyPath))
+            throw new ArgumentException($"File MainAssembly='{mainAssemblyPath}' doesn't exist.");
+        if (!File.Exists(WasmSettings.WasmMainJS))
+            throw new ArgumentException($"File MainJS='{WasmSettings.WasmMainJS}' doesn't exist.");
+
+        var paths = new List<string>();
+        assemblies = Directory.GetFiles(appDir, "*.dll");
+
+        // Create app
+        Directory.CreateDirectory(outputDir);
+        Directory.CreateDirectory(Path.Combine(outputDir, "managed"));
+        foreach (var assembly in assemblies)
+            File.Copy(assembly, Path.Combine(outputDir, "managed", Path.GetFileName(assembly)), true);
+
+        foreach (var f in new string[] { "dotnet.wasm", "dotnet.js" })
+            File.Copy(Path.Combine(appDir, f), Path.Combine(outputDir, f), true);
+
+        File.Copy(WasmSettings.WasmMainJS, Path.Combine(outputDir, "runtime.js"),  true);
+
+        using (var sw = File.CreateText(Path.Combine(outputDir, "mono-config.js")))
+        {
+            sw.WriteLine("config = {");
+            sw.WriteLine("\tvfs_prefix: \"managed\",");
+            sw.WriteLine("\tdeploy_prefix: \"managed\",");
+            sw.WriteLine("\tenable_debugging: 0,");
+            sw.WriteLine("\tassembly_list: [");
+            foreach (var assembly in assemblies)
+            {
+                sw.Write("\t\t\"" + Path.GetFileName(assembly) + "\"");
+                sw.WriteLine(",");
+            }
+            sw.WriteLine ("\t],");
+            sw.WriteLine("\tfiles_to_map: [],");
+
+            sw.WriteLine ("}");
+        }
+
+        return true;
+    }
+}