From 6dd26bebcbeb8aee058cd268b6fb12d2b9000a13 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 22 Oct 2020 12:40:59 +0200 Subject: [PATCH 1/3] don't remove the auto-generated files when the build fails --- src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 66ab60b07d..f139a12d8a 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -143,11 +143,11 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, var info = buildResults[benchmark]; var buildResult = info.buildResult; - if (!config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles)) - artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup); - if (buildResult.IsBuildSuccess) { + if (!config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles)) + artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup); + var report = RunCore(benchmark, info.benchmarkId, logger, resolver, buildResult); if (report.AllMeasurements.Any(m => m.Operations == 0)) throw new InvalidOperationException("An iteration with 'Operations == 0' detected"); From 97e2f9171c8de3b870e1998061e96dce8ba3ddd4 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 22 Oct 2020 12:41:20 +0200 Subject: [PATCH 2/3] print a nice error with a link to troubleshooting guide --- src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index f139a12d8a..00e87b1752 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -173,6 +173,14 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, else if (buildResult.ErrorMessage != null) logger.WriteLineError($"// Build Error: {buildResult.ErrorMessage}"); + if (!benchmark.Job.GetToolchain().IsInProcess) + { + logger.WriteLine(); + logger.WriteLineError("// BenchmarkDotNet has failed to build the auto-generated boilerplate code."); + logger.WriteLineError($"// It can be found in {buildResult.ArtifactsPaths.BuildArtifactsDirectoryPath}"); + logger.WriteLineError("// Please follow the troubleshooting guide: https://benchmarkdotnet.org/articles/guides/troubleshooting.html"); + } + if (config.Options.IsSet(ConfigOptions.StopOnFirstError)) break; } From fd688123126c8d013bf73fdded0c2b358dcd46c7 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 22 Oct 2020 13:10:33 +0200 Subject: [PATCH 3/3] update the troubleshooting guide --- docs/articles/guides/troubleshooting.md | 35 +++++++++++++++---------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/docs/articles/guides/troubleshooting.md b/docs/articles/guides/troubleshooting.md index f2e9743915..57e96f2e64 100644 --- a/docs/articles/guides/troubleshooting.md +++ b/docs/articles/guides/troubleshooting.md @@ -7,26 +7,33 @@ You need to be aware of the fact that to ensure process-level isolation Benchmar How do you know that BenchmarkDotNet has failed to build the project? BDN is going to tell you about it. An example: ```log - // ***** BenchmarkRunner: Start ***** - // ***** Building 1 exe(s) in Parallel: Start ***** - // msbuild /p:ConfigurationGroup=Release /p:UseSharedCompilation=false took 12,93s and exited with 1 - // ***** Done, took 00:00:15 (15.59 sec) ***** - // Found benchmarks: - // Perf_Console.OpenStandardInput: Job-ZAFVFJ(Force=True, Toolchain=CoreFX, IterationCount=3, LaunchCount=1, WarmupCount=3) - - // Build Exception: Microsoft (R) Build Engine version 15.9.8-preview+g0a5001fc4d for .NET Core - Copyright (C) Microsoft Corporation. All rights reserved. - - C:\Program Files\dotnet\sdk\2.2.100-preview2-009404\Microsoft.Common.CurrentVersion.targets(4176,5): warning MSB3026: Could not copy "C:\Projects\corefx/bin/obj/AnyOS.AnyCPU.Release/BenchmarksRunner/netstandard/BenchmarksRunner.exe" to "C:\Projects\corefx\bin/AnyOS.AnyCPU.Release/BenchmarksRunner/netstandard/BenchmarksRunner.exe". Beginning retry 1 in 1000ms. The process cannot access the file 'C:\Projects\corefx\bin\AnyOS.AnyCPU.Release\BenchmarksRunner\netstandard\BenchmarksRunner.exe' because it is being used by another process. [C:\Projects\corefx\src\Common\perf\BenchmarksRunner\BenchmarksRunner.csproj] +// Validating benchmarks: +// ***** BenchmarkRunner: Start ***** +// ***** Found 1 benchmark(s) in total ***** +// ***** Building 1 exe(s) in Parallel: Start ***** +// start dotnet restore /p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true /p:Optimize=true in C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\netcoreapp2.1\c6045772-d3c7-4dbe-ab37-4aca6dcb6ec4 +// command took 0.51s and exited with 1 +// ***** Done, took 00:00:00 (0.66 sec) ***** +// Found 1 benchmarks: +// IntroBasic.Sleep: DefaultJob + +// Build Error: Standard output: + + Standard error: + C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\netcoreapp2.1\c6045772-d3c7-4dbe-ab37-4aca6dcb6ec4\BenchmarkDotNet.Autogenerated.csproj(36,1): error MSB4025: The project file could not be loaded. Unexpected end of file while parsing Comment has occurred. Line 36, position 1. + +// BenchmarkDotNet has failed to build the auto-generated boilerplate code. +// It can be found in C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\netcoreapp2.1\c6045772-d3c7-4dbe-ab37-4aca6dcb6ec4 +// Please follow the troubleshooting guide: https://benchmarkdotnet.org/articles/guides/troubleshooting.html ``` If the error message is not clear enough, you need to investigate it further. How to troubleshoot the build process: -1. Configure BenchmarkDotNet to keep auto-generated benchmark files (they are being removed after benchmark is executed by default). You can do that by either passing `--keepFiles` console argument to `BenchmarkSwitcher` or by using `[KeepBenchmarkFiles]` attribute on the type which defines the benchmarks or by using `config.KeepBenchmarkFiles()` extension method. -2. Run the benchmarks -3. Go to the output folder, which typicaly is `bin\Release\$FrameworkMoniker` and search for the new folder with auto-generated files. The name of the folder is just Job's ID. So if you are using `--job short` the folder should be called "ShortRun". If you want to change the name, use `Job.WithId("$newID")` extension method. +1. Run the benchmarks. +2. Read the error message. If it does not contain the answer to your problem, please continue to the next step. +3. Go to the build artifacts folder (path printed by BDN). 4. The folder should contain: * a file with source code (ends with `.notcs` to make sure IDE don't include it in other projects by default) * a project file (`.csproj`)