Log a warning when an empty benchmark is run (#2718)

* Add message and throw a validation error + add test

* change the test to see if there is a log message

* Update the logic to throw a validation error + update tests + remove redundant check for any benchmark

* fix case of empty BenchmarkRunInfo array & added test & fix EventProcessorTests

* Fix tests error when running an empty RunInfo array & fix wakeLock logic accordinly

* Improve tests naming and precision

* update tests to support benchmark switcher + add logger + add better logs

* fix benchmarswitcher tests error

Author: AvishaiDotan

src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs

@@ -19,7 +19,6 @@
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Mathematics;
-using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.Parameters;
@@ -74,9 +73,6 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
                 if (validationErrors.Any(validationError => validationError.IsCritical))
                     return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath, validationErrors.ToImmutableArray()) };
 
-                if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any()))
-                    return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) };
-
                 eventProcessor.OnEndValidationStage();
 
                 int totalBenchmarkCount = supportedBenchmarks.Sum(benchmarkInfo => benchmarkInfo.BenchmarksCases.Length);
@@ -579,20 +575,47 @@ private static void LogTotalTime(ILogger logger, TimeSpan time, int executedBenc
         private static (BenchmarkRunInfo[], List<ValidationError>) GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, IResolver resolver)
         {
             List<ValidationError> validationErrors = new ();
+            List<BenchmarkRunInfo> runInfos = new (benchmarkRunInfos.Length);
+
+            if (benchmarkRunInfos.Length == 0)
+            {
+                validationErrors.Add(new ValidationError(true, $"No benchmarks were found."));
+                return (Array.Empty<BenchmarkRunInfo>(), validationErrors);
+            }
+
+            foreach (var benchmarkRunInfo in benchmarkRunInfos)
+            {
+                if (benchmarkRunInfo.BenchmarksCases.Length == 0)
+                {
+                    validationErrors.Add(new ValidationError(true, $"No [Benchmark] attribute found on '{benchmarkRunInfo.Type.Name}' benchmark case."));
+                    continue;
+                }
 
-            var runInfos = benchmarkRunInfos.Select(info => new BenchmarkRunInfo(
-                    info.BenchmarksCases.Where(benchmark =>
+                var validBenchmarks = benchmarkRunInfo.BenchmarksCases
+                    .Where(benchmark =>
                     {
-                        var errors = benchmark.GetToolchain().Validate(benchmark, resolver).ToArray();
+
+                        var errors = benchmark.GetToolchain()
+                            .Validate(benchmark, resolver)
+                            .ToArray();
+
                         validationErrors.AddRange(errors);
-                        return !errors.Any();
-                    }).ToArray(),
-                    info.Type,
-                    info.Config))
-                .Where(infos => infos.BenchmarksCases.Any())
-                .ToArray();
-
-            return (runInfos, validationErrors);
+
+                        return errors.Length == 0;
+                    })
+                    .ToArray();
+
+                runInfos.Add(
+                    new BenchmarkRunInfo(
+                        validBenchmarks,
+                        benchmarkRunInfo.Type,
+                        benchmarkRunInfo.Config
+
+                    ));
+
+
+            }
+            return (runInfos.ToArray(), validationErrors);
         }
 
         private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRunInfos)
@@ -665,6 +688,9 @@ void AddLogger(ILogger logger)
                 foreach (var logger in benchmarkRunInfo.Config.GetLoggers())
                     AddLogger(logger);
 
+            if (benchmarkRunInfos.Length == 0)
+                AddLogger(new ConsoleLogger());
+
             AddLogger(streamLogger);
 
             return new CompositeLogger(loggers.Values.ToImmutableHashSet());

src/BenchmarkDotNet/Running/TypeFilter.cs

@@ -15,6 +15,32 @@ public static (bool allTypesValid, IReadOnlyList<Type> runnable) GetTypesWithRun
         {
             var validRunnableTypes = new List<Type>();
 
+            bool hasRunnableTypeBenchmarks = types.Any(type => type.ContainsRunnableBenchmarks());
+            bool hasRunnableAssemblyBenchmarks = assemblies.Any(assembly => GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks()).Length > 0);
+
+            if (!hasRunnableTypeBenchmarks && !hasRunnableAssemblyBenchmarks)
+            {
+                if (types.Any())
+                {
+                    foreach (var type in types)
+                    {
+                        logger.WriteLineError($"No [Benchmark] attribute found on '{type.Name}' benchmark case.");
+                    }
+                }
+                else if (assemblies.Any())
+                {
+                    foreach (var assembly in assemblies)
+                    {
+                        logger.WriteLineError($"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly.");
+                    }
+                }
+                else
+                {
+                    logger.WriteLineError("No benchmarks were found.");
+                }
+                return (false, Array.Empty<Type>());
+            }
+
             foreach (var type in types)
             {
                 if (type.ContainsRunnableBenchmarks())

src/BenchmarkDotNet/Running/WakeLock.cs

@@ -11,7 +11,7 @@ namespace BenchmarkDotNet.Running;
 internal partial class WakeLock
 {
     public static WakeLockType GetWakeLockType(BenchmarkRunInfo[] benchmarkRunInfos) =>
-        benchmarkRunInfos.Select(static i => i.Config.WakeLock).Max();
+        benchmarkRunInfos.Length == 0 ? WakeLockType.None : benchmarkRunInfos.Select(static i => i.Config.WakeLock).Max();
 
     private static readonly bool OsVersionIsSupported =
         // Must be windows 7 or greater

tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs

@@ -66,7 +66,7 @@ public void WhenInvalidTypeIsProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAre
                 .Run(new[] { "--filter", "*" }, config);
 
             Assert.Empty(summaries);
-            Assert.Contains("Type BenchmarkDotNet.IntegrationTests.ClassC is invalid.", logger.GetLog());
+            Assert.Contains(GetValidationErrorForType(typeof(ClassC)), logger.GetLog());
         }
 
         [Fact]
@@ -80,7 +80,7 @@ public void WhenNoTypesAreProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAreExe
                 .Run(new[] { "--filter", "*" }, config);
 
             Assert.Empty(summaries);
-            Assert.Contains("No benchmarks to choose from. Make sure you provided public non-sealed non-static types with public [Benchmark] methods.", logger.GetLog());
+            Assert.Contains("No benchmarks were found.", logger.GetLog());
         }
 
         [Fact]
@@ -369,6 +369,11 @@ public IReadOnlyList<Type> AskUser(IReadOnlyList<Type> allTypes, ILogger logger)
                 return returnValue;
             }
         }
+
+        private string GetValidationErrorForType(Type type)
+        {
+            return $"No [Benchmark] attribute found on '{type.Name}' benchmark case.";
+        }
     }
 }
 
@@ -424,6 +429,8 @@ public override void ExportToLog(Summary summary, ILogger logger)
             exported = true;
         }
     }
+
+
 }
 
 namespace BenchmarkDotNet.NOTIntegrationTests

tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs

@@ -23,8 +23,9 @@ public class EventProcessorTests
         public void WhenUsingEventProcessorAndNoBenchmarks()
         {
             var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassEmpty) });
-            Assert.Single(events);
+            Assert.Equal(2, events.Count);
             Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType);
+            Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType);
         }
 
         [Fact]