Issue #1736: Add ExceptionDiagnoser (#2169)

Co-authored-by: Adam Sitnik <[email protected]>

Author: Serg046

docs/articles/configs/diagnosers.md

@@ -37,6 +37,7 @@ The current Diagnosers are:
   It is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS.
   Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.com/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details.
 - Threading Diagnoser (`ThreadingDiagnoser`) - .NET Core 3.0+ diagnoser that reports some Threading statistics.
+- Exception Diagnoser (`ExceptionDiagnoser`) - a diagnoser that reports the frequency of exceptions thrown during the operation.
 
 ## Usage
 
@@ -59,6 +60,7 @@ private class Config : ManualConfig
         Add(new InliningDiagnoser());
         Add(new EtwProfiler());
         Add(ThreadingDiagnoser.Default);
+        Add(ExceptionDiagnoser.Default);
     }
 }
 ```
@@ -72,6 +74,7 @@ You can also use one of the following attributes (apply it on a class that conta
 [ConcurrencyVisualizerProfiler]
 [NativeMemoryProfiler]
 [ThreadingDiagnoser]
+[ExceptionDiagnoser]
 ```
 
 In BenchmarkDotNet, 1kB = 1024B, 1MB = 1024kB, and so on. The column Gen X means number of GC collections per 1000 operations for that generation.
@@ -123,3 +126,5 @@ In BenchmarkDotNet, 1kB = 1024B, 1MB = 1024kB, and so on. The column Gen X means
 [!include[IntroNativeMemory](../samples/IntroNativeMemory.md)]
 
 [!include[IntroThreadingDiagnoser](../samples/IntroThreadingDiagnoser.md)]
+
+[!include[IntroExceptionDiagnoser](../samples/IntroExceptionDiagnoser.md)]

docs/articles/samples/IntroExceptionDiagnoser.md

@@ -0,0 +1,26 @@
+---
+uid: BenchmarkDotNet.Samples.IntroExceptionDiagnoser
+---
+
+## Sample: IntroExceptionDiagnoser
+
+The `ExceptionDiagnoser` uses [AppDomain.FirstChanceException](https://learn.microsoft.com/en-us/dotnet/api/system.appdomain.firstchanceexception) API to report:
+
+* Exception frequency: The number of exceptions thrown during the operations divided by the number of operations.
+
+### Source code
+
+[!code-csharp[IntroExceptionDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs)]
+
+### Output
+
+|                 Method |     Mean |     Error |    StdDev | Exception frequency |
+|----------------------- |---------:|----------:|----------:|--------------------:|
+| ThrowExceptionRandomly | 4.936 us | 0.1542 us | 0.4499 us |              0.1381 |
+
+### Links
+
+* @docs.diagnosers
+* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroExceptionDiagnoser
+
+---

samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs

@@ -0,0 +1,22 @@
+using BenchmarkDotNet.Attributes;
+using System;
+
+namespace BenchmarkDotNet.Samples
+{
+    [ExceptionDiagnoser]
+    public class IntroExceptionDiagnoser
+    {
+        [Benchmark]
+        public void ThrowExceptionRandomly()
+        {
+            try
+            {
+                if (new Random().Next(0, 5) > 1)
+                {
+                    throw new Exception();
+                }
+            }
+            catch { }
+        }
+    }
+}

src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs

@@ -0,0 +1,14 @@
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using System;
+
+namespace BenchmarkDotNet.Attributes
+{
+    [AttributeUsage(AttributeTargets.Class)]
+    public class ExceptionDiagnoserAttribute : Attribute, IConfigSource
+    {
+        public IConfig Config { get; }
+
+        public ExceptionDiagnoserAttribute() => Config = ManualConfig.CreateEmpty().AddDiagnoser(ExceptionDiagnoser.Default);
+    }
+}

src/BenchmarkDotNet/Columns/Column.cs

@@ -55,6 +55,7 @@ public static class Column
         public const string CompletedWorkItems = "Completed Work Items";
         public const string LockContentions = "Lock Contentions";
         public const string CodeSize = "Code Size";
+        public const string Exceptions = "Exceptions";
 
         //Characteristics:
         public const string Id = "Id";

src/BenchmarkDotNet/Configs/ImmutableConfig.cs

@@ -106,9 +106,11 @@ internal ImmutableConfig(
 
         public bool HasThreadingDiagnoser() => diagnosers.Contains(ThreadingDiagnoser.Default);
 
+        public bool HasExceptionDiagnoser() => diagnosers.Contains(ExceptionDiagnoser.Default);
+
         internal bool HasPerfCollectProfiler() => diagnosers.OfType<PerfCollectProfiler>().Any();
 
-        public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser();
+        public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser() || HasExceptionDiagnoser();
 
         public IDiagnoser GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode)
         {

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -37,6 +37,9 @@ public class CommandLineOptions
         [Option('t', "threading", Required = false, Default = false, HelpText = "Prints threading statistics")]
         public bool UseThreadingDiagnoser { get; set; }
 
+        [Option("exceptions", Required = false, Default = false, HelpText = "Prints exception statistics")]
+        public bool UseExceptionDiagnoser { get; set; }
+
         [Option('d', "disasm", Required = false, Default = false, HelpText = "Gets disassembly of benchmarked code")]
         public bool UseDisassemblyDiagnoser
         {

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -213,6 +213,8 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo
                 config.AddDiagnoser(MemoryDiagnoser.Default);
             if (options.UseThreadingDiagnoser)
                 config.AddDiagnoser(ThreadingDiagnoser.Default);
+            if (options.UseExceptionDiagnoser)
+                config.AddDiagnoser(ExceptionDiagnoser.Default);
             if (options.UseDisassemblyDiagnoser)
                 config.AddDiagnoser(new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig(
                     maxDepth: options.DisassemblerRecursiveDepth,

src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs

@@ -15,6 +15,7 @@ public DiagnoserResults(BenchmarkCase benchmarkCase, ExecuteResult executeResult
             GcStats = executeResult.GcStats;
             ThreadingStats = executeResult.ThreadingStats;
             BuildResult = buildResult;
+            ExceptionFrequency = executeResult.ExceptionFrequency;
         }
 
         public BenchmarkCase BenchmarkCase { get; }
@@ -25,6 +26,8 @@ public DiagnoserResults(BenchmarkCase benchmarkCase, ExecuteResult executeResult
 
         public ThreadingStats ThreadingStats { get; }
 
+        public double ExceptionFrequency { get; }
+
         public BuildResult BuildResult { get; }
     }
 }

src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs

@@ -0,0 +1,54 @@
+using BenchmarkDotNet.Analysers;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Validators;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace BenchmarkDotNet.Diagnosers
+{
+    public class ExceptionDiagnoser : IDiagnoser
+    {
+        public static readonly ExceptionDiagnoser Default = new ExceptionDiagnoser();
+
+        private ExceptionDiagnoser() { }
+
+        public IEnumerable<string> Ids => new[] { nameof(ExceptionDiagnoser) };
+
+        public IEnumerable<IExporter> Exporters => Array.Empty<IExporter>();
+
+        public IEnumerable<IAnalyser> Analysers => Array.Empty<IAnalyser>();
+
+        public void DisplayResults(ILogger logger) { }
+
+        public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead;
+
+        public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
+
+        public IEnumerable<Metric> ProcessResults(DiagnoserResults results)
+        {
+            yield return new Metric(ExceptionsFrequencyMetricDescriptor.Instance, results.ExceptionFrequency);
+        }
+
+        public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => Enumerable.Empty<ValidationError>();
+
+        private class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor
+        {
+            internal static readonly IMetricDescriptor Instance = new ExceptionsFrequencyMetricDescriptor();
+
+            public string Id => "ExceptionFrequency";
+            public string DisplayName => Column.Exceptions;
+            public string Legend => "Exceptions thrown per single operation";
+            public string NumberFormat => "#0.0000";
+            public UnitType UnitType => UnitType.Dimensionless;
+            public string Unit => "Count";
+            public bool TheGreaterTheBetter => false;
+            public int PriorityInCategory => 0;
+        }
+    }
+}

src/BenchmarkDotNet/Engines/Engine.cs

@@ -132,16 +132,16 @@ public RunResults Run()
 
             Host.AfterMainRun();
 
-            (GcStats workGcHasDone, ThreadingStats threadingStats) = includeExtraStats
+            (GcStats workGcHasDone, ThreadingStats threadingStats, double exceptionFrequency) = includeExtraStats
                 ? GetExtraStats(new IterationData(IterationMode.Workload, IterationStage.Actual, 0, invokeCount, UnrollFactor))
-                : (GcStats.Empty, ThreadingStats.Empty);
+                : (GcStats.Empty, ThreadingStats.Empty, 0);
 
             if (EngineEventSource.Log.IsEnabled())
                 EngineEventSource.Log.BenchmarkStop(BenchmarkName);
 
             var outlierMode = TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, Resolver);
 
-            return new RunResults(idle, main, outlierMode, workGcHasDone, threadingStats);
+            return new RunResults(idle, main, outlierMode, workGcHasDone, threadingStats, exceptionFrequency);
         }
 
         public Measurement RunIteration(IterationData data)
@@ -189,7 +189,7 @@ public Measurement RunIteration(IterationData data)
             return measurement;
         }
 
-        private (GcStats, ThreadingStats) GetExtraStats(IterationData data)
+        private (GcStats, ThreadingStats, double) GetExtraStats(IterationData data)
         {
             // we enable monitoring after main target run, for this single iteration which is executed at the end
             // so even if we enable AppDomain monitoring in separate process
@@ -199,19 +199,23 @@ public Measurement RunIteration(IterationData data)
             IterationSetupAction(); // we run iteration setup first, so even if it allocates, it is not included in the results
 
             var initialThreadingStats = ThreadingStats.ReadInitial(); // this method might allocate
+            var exceptionsStats = new ExceptionsStats(); // allocates
+            exceptionsStats.StartListening(); // this method might allocate
             var initialGcStats = GcStats.ReadInitial();
 
             WorkloadAction(data.InvokeCount / data.UnrollFactor);
 
+            exceptionsStats.Stop();
             var finalGcStats = GcStats.ReadFinal();
             var finalThreadingStats = ThreadingStats.ReadFinal();
 
             IterationCleanupAction(); // we run iteration cleanup after collecting GC stats
 
-            GcStats gcStats = (finalGcStats - initialGcStats).WithTotalOperations(data.InvokeCount * OperationsPerInvoke);
+            var totalOperationsCount = data.InvokeCount * OperationsPerInvoke;
+            GcStats gcStats = (finalGcStats - initialGcStats).WithTotalOperations(totalOperationsCount);
             ThreadingStats threadingStats = (finalThreadingStats - initialThreadingStats).WithTotalOperations(data.InvokeCount * OperationsPerInvoke);
 
-            return (gcStats, threadingStats);
+            return (gcStats, threadingStats, exceptionsStats.ExceptionsCount / (double)totalOperationsCount);
         }
 
         [MethodImpl(MethodImplOptions.NoInlining)]

src/BenchmarkDotNet/Engines/ExceptionsStats.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Runtime.ExceptionServices;
+
+namespace BenchmarkDotNet.Engines
+{
+    internal class ExceptionsStats
+    {
+        internal const string ResultsLinePrefix = "// Exceptions: ";
+
+        internal ulong ExceptionsCount { get; private set; }
+
+        public void StartListening()
+        {
+            AppDomain.CurrentDomain.FirstChanceException += OnFirstChanceException;
+        }
+
+        public void Stop()
+        {
+            AppDomain.CurrentDomain.FirstChanceException -= OnFirstChanceException;
+        }
+
+        private void OnFirstChanceException(object sender, FirstChanceExceptionEventArgs e)
+        {
+            ExceptionsCount++;
+        }
+
+        public static string ToOutputLine(double exceptionCount) => $"{ResultsLinePrefix} {exceptionCount}";
+
+        public static double Parse(string line)
+        {
+            if (!line.StartsWith(ResultsLinePrefix))
+                throw new NotSupportedException($"Line must start with {ResultsLinePrefix}");
+
+            var measurement = line.Remove(0, ResultsLinePrefix.Length);
+            if (!double.TryParse(measurement, out var exceptionsNumber))
+            {
+                throw new NotSupportedException("Invalid string");
+            }
+
+            return exceptionsNumber;
+        }
+    }
+}

src/BenchmarkDotNet/Engines/RunResults.cs

@@ -23,17 +23,21 @@ public struct RunResults
 
         public ThreadingStats ThreadingStats { get; }
 
+        public double ExceptionFrequency { get; }
+
         public RunResults([CanBeNull] IReadOnlyList<Measurement> overhead,
                           [NotNull] IReadOnlyList<Measurement> workload,
                           OutlierMode outlierMode,
                           GcStats gcStats,
-                          ThreadingStats threadingStats)
+                          ThreadingStats threadingStats,
+                          double exceptionFrequency)
         {
             this.outlierMode = outlierMode;
             Overhead = overhead;
             Workload = workload;
             GCStats = gcStats;
             ThreadingStats = threadingStats;
+            ExceptionFrequency = exceptionFrequency;
         }
 
         public IEnumerable<Measurement> GetMeasurements()
@@ -68,6 +72,8 @@ public void Print(TextWriter outWriter)
                 outWriter.WriteLine(GCStats.ToOutputLine());
             if (!ThreadingStats.Equals(ThreadingStats.Empty))
                 outWriter.WriteLine(ThreadingStats.ToOutputLine());
+            if (ExceptionFrequency > 0)
+                outWriter.WriteLine(ExceptionsStats.ToOutputLine(ExceptionFrequency));
 
             outWriter.WriteLine();
         }