From e400da2bc5a4b63e98df66c82b567ed8238e79c7 Mon Sep 17 00:00:00 2001 From: Avishai Dotan <108017307+AvishaiDotan@users.noreply.github.com> Date: Thu, 17 Apr 2025 21:51:55 +0300 Subject: [PATCH 1/8] Add message and throw a validation error + add test --- .../Running/BenchmarkRunnerClean.cs | 3 ++ .../Running/RunningEmptyBenchmarkTests.cs | 36 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 6d9d48c3cc..f5f0acdfe2 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -74,7 +74,10 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath, validationErrors.ToImmutableArray()) }; if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any())) + { + compositeLogger.WriteLineError("// No benchmarks have been found"); return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) }; + } eventProcessor.OnEndValidationStage(); diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs new file mode 100644 index 0000000000..bf9ace55bd --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -0,0 +1,36 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Parameters; +using System.Reflection; +using Xunit; +using Xunit.Abstractions; +using BenchmarkDotNet.Tests.XUnit; + +namespace BenchmarkDotNet.Tests.Running +{ + public class RunningEmptyBenchmarkTests + { + [Fact] + public void WhenNoBenchmarksAreFound_ReturnsNoBenchmarksHaveBeenFoundValidationError() + { + var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmarks), null) }); + // Act + // Assert + Assert.Single(summaries); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Equal("No benchmarks have been found", summary.ValidationErrors[0].Message); + Assert.True(summary.ValidationErrors[0].IsCritical); + } + + public class EmptyBenchmarks + { + // No benchmark methods here + } + } +} \ No newline at end of file From 716a56d863909c5ae2cccd430c707fcef60e2c77 Mon Sep 17 00:00:00 2001 From: Avishai Dotan <108017307+AvishaiDotan@users.noreply.github.com> Date: Thu, 17 Apr 2025 22:01:44 +0300 Subject: [PATCH 2/8] change the test to see if there is a log message --- .../Running/RunningEmptyBenchmarkTests.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs index bf9ace55bd..11784d8824 100644 --- a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -10,6 +10,7 @@ using Xunit; using Xunit.Abstractions; using BenchmarkDotNet.Tests.XUnit; +using BenchmarkDotNet.Loggers; namespace BenchmarkDotNet.Tests.Running { @@ -18,14 +19,11 @@ public class RunningEmptyBenchmarkTests [Fact] public void WhenNoBenchmarksAreFound_ReturnsNoBenchmarksHaveBeenFoundValidationError() { - var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmarks), null) }); - // Act - // Assert - Assert.Single(summaries); - var summary = summaries[0]; - Assert.True(summary.HasCriticalValidationErrors); - Assert.Equal("No benchmarks have been found", summary.ValidationErrors[0].Message); - Assert.True(summary.ValidationErrors[0].IsCritical); + var logger = new AccumulationLogger(); + var config = ManualConfig.CreateEmpty().AddLogger(logger); + var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmarks), config) }); + Console.WriteLine(logger.GetLog()); + Assert.Contains("// No benchmarks have been found", logger.GetLog()); } public class EmptyBenchmarks From bdf1d866d6bb8c8f610346fe2946782c68854bd0 Mon Sep 17 00:00:00 2001 From: Avishai Dotan <108017307+AvishaiDotan@users.noreply.github.com> Date: Sat, 19 Apr 2025 22:58:52 +0300 Subject: [PATCH 3/8] Update the logic to throw a validation error + update tests + remove redundant check for any benchmark --- .../Running/BenchmarkRunnerClean.cs | 50 +++++++++++------- .../Running/RunningEmptyBenchmarkTests.cs | 52 ++++++++++++++++--- 2 files changed, 76 insertions(+), 26 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index f5f0acdfe2..77e07d4702 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -19,7 +19,6 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Mathematics; -using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.Parameters; @@ -73,12 +72,6 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) if (validationErrors.Any(validationError => validationError.IsCritical)) return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath, validationErrors.ToImmutableArray()) }; - if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any())) - { - compositeLogger.WriteLineError("// No benchmarks have been found"); - return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) }; - } - eventProcessor.OnEndValidationStage(); int totalBenchmarkCount = supportedBenchmarks.Sum(benchmarkInfo => benchmarkInfo.BenchmarksCases.Length); @@ -581,20 +574,41 @@ private static void LogTotalTime(ILogger logger, TimeSpan time, int executedBenc private static (BenchmarkRunInfo[], List) GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, IResolver resolver) { List validationErrors = new (); + List runInfos = new (benchmarkRunInfos.Length); - var runInfos = benchmarkRunInfos.Select(info => new BenchmarkRunInfo( - info.BenchmarksCases.Where(benchmark => + foreach (var benchmarkRunInfo in benchmarkRunInfos) + { + if (benchmarkRunInfo.BenchmarksCases.Length == 0) + { + validationErrors.Add(new ValidationError(true, $"No [Benchmark] attribute found on '{benchmarkRunInfo.Type.Name}' benchmark case.")); + continue; + } + + var validBenchmarks = benchmarkRunInfo.BenchmarksCases + .Where(benchmark => { - var errors = benchmark.GetToolchain().Validate(benchmark, resolver).ToArray(); + + var errors = benchmark.GetToolchain() + .Validate(benchmark, resolver) + .ToArray(); + validationErrors.AddRange(errors); - return !errors.Any(); - }).ToArray(), - info.Type, - info.Config)) - .Where(infos => infos.BenchmarksCases.Any()) - .ToArray(); - - return (runInfos, validationErrors); + + return errors.Length == 0; + }) + .ToArray(); + + runInfos.Add( + new BenchmarkRunInfo( + validBenchmarks, + benchmarkRunInfo.Type, + benchmarkRunInfo.Config + + )); + + + } + return (runInfos.ToArray(), validationErrors); } private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRunInfos) diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs index 11784d8824..b023ca7250 100644 --- a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -11,24 +11,60 @@ using Xunit.Abstractions; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Loggers; +using System.Linq; namespace BenchmarkDotNet.Tests.Running { public class RunningEmptyBenchmarkTests { [Fact] - public void WhenNoBenchmarksAreFound_ReturnsNoBenchmarksHaveBeenFoundValidationError() + public void WhenRunningSingleEmptyBenchmark_ValidationErrorIsThrown() { - var logger = new AccumulationLogger(); - var config = ManualConfig.CreateEmpty().AddLogger(logger); - var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmarks), config) }); - Console.WriteLine(logger.GetLog()); - Assert.Contains("// No benchmarks have been found", logger.GetLog()); + var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null) }); + Assert.Single(summaries); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Single(summary.ValidationErrors); + Assert.Equal($"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case.", summary.ValidationErrors[0].Message); } - public class EmptyBenchmarks + [Fact] + public void WhenRunningMultipleEmptyBenchmarks_ValidationErrorIsThrown() + { + var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null), BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null) }); + Assert.Single(summaries); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Equal(2, summary.ValidationErrors.Count()); + Assert.Equal($"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case.", summary.ValidationErrors[0].Message); + Assert.Equal($"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case.", summary.ValidationErrors[1].Message); + } + + [Fact] + public void WhenRunningMultipleBenchmarksOneOfWhichIsEmpty_ValidationErrorIsThrown() + { + var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null), BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), null) }); + Assert.Single(summaries); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == $"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case."); + } + + public class EmptyBenchmark + { + } + + public class NotEmptyBenchmark { - // No benchmark methods here + [Benchmark] + public void Benchmark() + { + var sum = 0; + for (int i = 0; i < 1; i++) + { + sum += i; + } + } } } } \ No newline at end of file From 9bd026687b6aad6a522e0ccc1a098aa64cdef3b4 Mon Sep 17 00:00:00 2001 From: Avishai Dotan <108017307+AvishaiDotan@users.noreply.github.com> Date: Sun, 20 Apr 2025 09:19:21 +0300 Subject: [PATCH 4/8] fix case of empty BenchmarkRunInfo array & added test & fix EventProcessorTests --- src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs | 6 ++++++ .../EventProcessorTests.cs | 3 ++- .../Running/RunningEmptyBenchmarkTests.cs | 11 +++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 77e07d4702..5f7d3c8a10 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -576,6 +576,12 @@ private static (BenchmarkRunInfo[], List) GetSupportedBenchmark List validationErrors = new (); List runInfos = new (benchmarkRunInfos.Length); + if (benchmarkRunInfos.Length == 0) + { + validationErrors.Add(new ValidationError(true, $"No benchmarks were found.")); + return (Array.Empty(), validationErrors); + } + foreach (var benchmarkRunInfo in benchmarkRunInfos) { if (benchmarkRunInfo.BenchmarksCases.Length == 0) diff --git a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs index 4cede6835b..f9272edd2d 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs @@ -23,8 +23,9 @@ public class EventProcessorTests public void WhenUsingEventProcessorAndNoBenchmarks() { var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassEmpty) }); - Assert.Single(events); + Assert.Equal(2, events.Count); Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType); } [Fact] diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs index b023ca7250..acabc87b45 100644 --- a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -50,6 +50,17 @@ public void WhenRunningMultipleBenchmarksOneOfWhichIsEmpty_ValidationErrorIsThro Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == $"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case."); } + [Fact] + public void WhenRunningEmptyArrayOfBenchmarks_ValidationErrorIsThrown() + { + var summaries = BenchmarkRunnerClean.Run(Array.Empty()); + Assert.Single(summaries); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Single(summary.ValidationErrors); + Assert.Equal("No benchmarks were found.", summary.ValidationErrors[0].Message); + } + public class EmptyBenchmark { } From 4e030e003f581b92a8fa0c3818a41e1b362be558 Mon Sep 17 00:00:00 2001 From: Avishai Dotan <108017307+AvishaiDotan@users.noreply.github.com> Date: Sun, 20 Apr 2025 11:19:12 +0300 Subject: [PATCH 5/8] Fix tests error when running an empty RunInfo array & fix wakeLock logic accordinly --- src/BenchmarkDotNet/Running/WakeLock.cs | 2 +- .../Running/RunningEmptyBenchmarkTests.cs | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/BenchmarkDotNet/Running/WakeLock.cs b/src/BenchmarkDotNet/Running/WakeLock.cs index a748f24626..124fbd5e22 100644 --- a/src/BenchmarkDotNet/Running/WakeLock.cs +++ b/src/BenchmarkDotNet/Running/WakeLock.cs @@ -11,7 +11,7 @@ namespace BenchmarkDotNet.Running; internal partial class WakeLock { public static WakeLockType GetWakeLockType(BenchmarkRunInfo[] benchmarkRunInfos) => - benchmarkRunInfos.Select(static i => i.Config.WakeLock).Max(); + benchmarkRunInfos.Length == 0 ? WakeLockType.None : benchmarkRunInfos.Select(static i => i.Config.WakeLock).Max(); private static readonly bool OsVersionIsSupported = // Must be windows 7 or greater diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs index acabc87b45..279bdfaf21 100644 --- a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -1,16 +1,7 @@ using System; using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; -using BenchmarkDotNet.Tests.Loggers; -using BenchmarkDotNet.Parameters; -using System.Reflection; using Xunit; -using Xunit.Abstractions; -using BenchmarkDotNet.Tests.XUnit; -using BenchmarkDotNet.Loggers; using System.Linq; namespace BenchmarkDotNet.Tests.Running From 34307c8982d5bfce6ab8a259cbe42000dcb71b16 Mon Sep 17 00:00:00 2001 From: Avishai Dotan <108017307+AvishaiDotan@users.noreply.github.com> Date: Mon, 21 Apr 2025 09:12:02 +0300 Subject: [PATCH 6/8] Improve tests naming and precision --- .../Running/RunningEmptyBenchmarkTests.cs | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs index 279bdfaf21..fb9e680fc6 100644 --- a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -9,30 +9,28 @@ namespace BenchmarkDotNet.Tests.Running public class RunningEmptyBenchmarkTests { [Fact] - public void WhenRunningSingleEmptyBenchmark_ValidationErrorIsThrown() + public void WhenRunningSingleEmptyBenchmark_NoBenchmarkValidationErrorIsThrown() { var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null) }); Assert.Single(summaries); var summary = summaries[0]; Assert.True(summary.HasCriticalValidationErrors); - Assert.Single(summary.ValidationErrors); - Assert.Equal($"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case.", summary.ValidationErrors[0].Message); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == $"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case."); } [Fact] - public void WhenRunningMultipleEmptyBenchmarks_ValidationErrorIsThrown() + public void WhenRunningMultipleEmptyBenchmarks_NoBenchmarkValidationErrorIsThrown() { - var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null), BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null) }); + var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null), BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark2), null) }); Assert.Single(summaries); var summary = summaries[0]; Assert.True(summary.HasCriticalValidationErrors); - Assert.Equal(2, summary.ValidationErrors.Count()); - Assert.Equal($"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case.", summary.ValidationErrors[0].Message); - Assert.Equal($"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case.", summary.ValidationErrors[1].Message); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == $"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case."); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == $"No [Benchmark] attribute found on '{typeof(EmptyBenchmark2).Name}' benchmark case."); } [Fact] - public void WhenRunningMultipleBenchmarksOneOfWhichIsEmpty_ValidationErrorIsThrown() + public void WhenRunningMultipleBenchmarksOneOfWhichIsEmpty_NoBenchmarkValidationErrorIsThrown() { var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null), BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), null) }); Assert.Single(summaries); @@ -42,20 +40,23 @@ public void WhenRunningMultipleBenchmarksOneOfWhichIsEmpty_ValidationErrorIsThro } [Fact] - public void WhenRunningEmptyArrayOfBenchmarks_ValidationErrorIsThrown() + public void WhenRunningEmptyArrayOfBenchmarks_NoBenchmarkValidationErrorIsThrown() { var summaries = BenchmarkRunnerClean.Run(Array.Empty()); Assert.Single(summaries); var summary = summaries[0]; Assert.True(summary.HasCriticalValidationErrors); - Assert.Single(summary.ValidationErrors); - Assert.Equal("No benchmarks were found.", summary.ValidationErrors[0].Message); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == "No benchmarks were found."); } public class EmptyBenchmark { } + public class EmptyBenchmark2 + { + } + public class NotEmptyBenchmark { [Benchmark] From 1e3a0dbda4e443a79eff0b17ffeada21f7157ff9 Mon Sep 17 00:00:00 2001 From: Avishai Dotan <108017307+AvishaiDotan@users.noreply.github.com> Date: Mon, 28 Apr 2025 22:03:03 +0300 Subject: [PATCH 7/8] update tests to support benchmark switcher + add logger + add better logs --- .../Running/BenchmarkRunnerClean.cs | 3 + src/BenchmarkDotNet/Running/TypeFilter.cs | 26 ++ .../Running/RunningEmptyBenchmarkTests.cs | 330 ++++++++++++++++-- 3 files changed, 337 insertions(+), 22 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 21cd5a6423..e29cc22f28 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -688,6 +688,9 @@ void AddLogger(ILogger logger) foreach (var logger in benchmarkRunInfo.Config.GetLoggers()) AddLogger(logger); + if (benchmarkRunInfos.Length == 0) + AddLogger(new ConsoleLogger()); + AddLogger(streamLogger); return new CompositeLogger(loggers.Values.ToImmutableHashSet()); diff --git a/src/BenchmarkDotNet/Running/TypeFilter.cs b/src/BenchmarkDotNet/Running/TypeFilter.cs index 4e58ebdeaf..741ec3fe3f 100644 --- a/src/BenchmarkDotNet/Running/TypeFilter.cs +++ b/src/BenchmarkDotNet/Running/TypeFilter.cs @@ -15,6 +15,32 @@ public static (bool allTypesValid, IReadOnlyList runnable) GetTypesWithRun { var validRunnableTypes = new List(); + bool hasRunnableTypeBenchmarks = types.Any(type => type.ContainsRunnableBenchmarks()); + bool hasRunnableAssemblyBenchmarks = assemblies.Any(assembly => GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks()).Length > 0); + + if (!hasRunnableTypeBenchmarks && !hasRunnableAssemblyBenchmarks) + { + if (types.Any()) + { + foreach (var type in types) + { + logger.WriteLineError($"No [Benchmark] attribute found on '{type.Name}' benchmark case."); + } + return (false, Array.Empty()); + } + else if (assemblies.Any()) + { + foreach (var assembly in assemblies) + { + logger.WriteLineError($"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly."); + } + return (false, Array.Empty()); + } + logger.WriteLineError("No benchmarks were found."); + return (false, Array.Empty()); + + } + foreach (var type in types) { if (type.ContainsRunnableBenchmarks()) diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs index fb9e680fc6..05ce92b2cf 100644 --- a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -2,53 +2,337 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Running; using Xunit; -using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using System.Runtime.InteropServices; namespace BenchmarkDotNet.Tests.Running { public class RunningEmptyBenchmarkTests { - [Fact] - public void WhenRunningSingleEmptyBenchmark_NoBenchmarkValidationErrorIsThrown() + #region BenchmarkRunner Methods Overview + /* + * Available BenchmarkRunner.Run methods: + * 1. Generic Type: + * - BenchmarkRunner.Run(IConfig? config = null, string[]? args = null) + * 2. Type-based: + * - BenchmarkRunner.Run(Type type, IConfig? config = null, string[]? args = null) + * - BenchmarkRunner.Run(Type[] types, IConfig? config = null, string[]? args = null) + * - BenchmarkRunner.Run(Type type, MethodInfo[] methods, IConfig? config = null) + * 3. Assembly-based: + * - BenchmarkRunner.Run(Assembly assembly, IConfig? config = null, string[]? args = null) + * 4. BenchmarkRunInfo-based: + * - BenchmarkRunner.Run(BenchmarkRunInfo benchmarkRunInfo) + * - BenchmarkRunner.Run(BenchmarkRunInfo[] benchmarkRunInfos) + * 5. Deprecated methods: + * - BenchmarkRunner.RunUrl(string url, IConfig? config = null) + * - BenchmarkRunner.RunSource(string source, IConfig? config = null) + */ + #endregion + #region Generic Type Tests + /// + /// Tests for BenchmarkRunner.Run method + /// + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void GenericTypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) { - var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null) }); - Assert.Single(summaries); - var summary = summaries[0]; + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(config, args); + if (args == null) + { + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + } + + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void GenericTypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(config, args); + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region Type-based Tests + /// + /// Tests for BenchmarkRunner.Run(Type) method + /// + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void TypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + + var summary = BenchmarkRunner.Run(typeof(EmptyBenchmark), config, args); Assert.True(summary.HasCriticalValidationErrors); - Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == $"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case."); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); } - [Fact] - public void WhenRunningMultipleEmptyBenchmarks_NoBenchmarkValidationErrorIsThrown() + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) { - var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null), BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark2), null) }); - Assert.Single(summaries); + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(typeof(NotEmptyBenchmark), config, args); + Assert.False(summaries.HasCriticalValidationErrors); + Assert.DoesNotContain(summaries.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + + /// + /// Tests for BenchmarkRunner.Run(Type[]) method + /// + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypesWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(EmptyBenchmark), typeof(EmptyBenchmark2) }, config, args); + if (args != null) + { + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + else + { + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark2))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + + + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypesWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(NotEmptyBenchmark) }, config, args); var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region BenchmarkRunInfo Tests + /// + /// Tests for BenchmarkRunner.Run(BenchmarkRunInfo) method + /// + [Fact] + public void BenchmarkRunInfoWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute() + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config)); Assert.True(summary.HasCriticalValidationErrors); - Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == $"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case."); - Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == $"No [Benchmark] attribute found on '{typeof(EmptyBenchmark2).Name}' benchmark case."); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Fact] + public void BenchmarkRunInfoWithBenchmarkAttribute_RunsSuccessfully() + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config)); + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); } + /// + /// Tests for BenchmarkRunner.Run(BenchmarkRunInfo[]) method + /// [Fact] - public void WhenRunningMultipleBenchmarksOneOfWhichIsEmpty_NoBenchmarkValidationErrorIsThrown() + public void BenchmarkRunInfosWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute() { - var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), null), BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), null) }); - Assert.Single(summaries); + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { + BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config), + BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark2), config) + }); var summary = summaries[0]; Assert.True(summary.HasCriticalValidationErrors); - Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == $"No [Benchmark] attribute found on '{typeof(EmptyBenchmark).Name}' benchmark case."); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark2))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); } [Fact] - public void WhenRunningEmptyArrayOfBenchmarks_NoBenchmarkValidationErrorIsThrown() + public void BenchmarkRunInfosWithBenchmarkAttribute_RunsSuccessfully() { - var summaries = BenchmarkRunnerClean.Run(Array.Empty()); - Assert.Single(summaries); + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config) }); var summary = summaries[0]; - Assert.True(summary.HasCriticalValidationErrors); - Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == "No benchmarks were found."); + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region Mixed Types Tests + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void MixedTypes_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(EmptyBenchmark), typeof(NotEmptyBenchmark) }, config, args); + if (args != null) + { + Assert.Contains(GetExpandedValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + else + { + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + } + #endregion + #region Assembly Tests + // In this tests there is no config and logger because the logger is initiated at CreateCompositeLogger when the BenchmarkRunInfo[] is empty + // those cannot be inserted using config + [Theory] + + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void AssemblyWithoutBenchmarks_ThrowsValidationError_WhenNoBenchmarksFound(string[]? args) + { + + // Create a mock assembly with no benchmark types + var assemblyName = new AssemblyName("MockAssembly"); + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule("MockModule"); + // Create a simple type in the assembly (no benchmarks) + var typeBuilder = moduleBuilder.DefineType("MockType", TypeAttributes.Public); + typeBuilder.CreateType(); + + Summary[] summaries = null; + if (args != null) + { + GetConfigWithLogger(out var logger, out var config); + summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + Assert.Contains(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); + } + else + { + summaries = BenchmarkRunner.Run(assemblyBuilder, null, args); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); + } + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void AssemblyWithBenchmarks_RunsSuccessfully_WhenBenchmarkAttributePresent(string[]? args) + { + // Skip test on .NET Framework 4.6.2 + if (RuntimeInformation.FrameworkDescription.Contains(".NET Framework 4")) + return; + + // Create a mock assembly with benchmark types + var assemblyName = new AssemblyName("MockAssemblyWithBenchmarks"); + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule("MockModule"); + + // Create a benchmark type + var benchmarkTypeBuilder = moduleBuilder.DefineType("MockBenchmark", TypeAttributes.Public); + var benchmarkMethod = benchmarkTypeBuilder.DefineMethod("Benchmark", MethodAttributes.Public, typeof(void), Type.EmptyTypes); + + // Generate method body + var ilGenerator = benchmarkMethod.GetILGenerator(); + ilGenerator.Emit(OpCodes.Ret); // Just return from the method + + var benchmarkAttributeCtor = typeof(BenchmarkAttribute).GetConstructor(new[] { typeof(int), typeof(string) }); + if (benchmarkAttributeCtor == null) + throw new InvalidOperationException("Could not find BenchmarkAttribute constructor"); + benchmarkMethod.SetCustomAttribute(new CustomAttributeBuilder( + benchmarkAttributeCtor, + new object[] { 0, "" })); + benchmarkTypeBuilder.CreateType(); + + Summary[] summaries = null; + if (args != null) + { + GetConfigWithLogger(out var logger, out var config); + summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + Assert.DoesNotContain(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); + } + else + { + summaries = BenchmarkRunner.Run(assemblyBuilder); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); + } + } + #endregion + #region Helper Methods + private string GetValidationErrorForType(Type type) + { + return $"No [Benchmark] attribute found on '{type.Name}' benchmark case."; + } + + private string GetAssemblylValidationError(Assembly assembly) + { + return $"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly."; + } + + private string GetExpandedValidationErrorForType(Type type) + { + return $"Type {type} is invalid. Only public, non-generic (closed generic types with public parameterless ctors are supported), non-abstract, non-sealed, non-static types with public instance [Benchmark] method(s) are supported."; + } + + private string GetGeneralValidationError() + { + return $"No benchmarks were found."; + } + + private void GetConfigWithLogger(out AccumulationLogger logger, out ManualConfig manualConfig) + { + logger = new AccumulationLogger(); + manualConfig = ManualConfig.CreateEmpty() + .AddLogger(logger) + .AddColumnProvider(DefaultColumnProviders.Instance); } + #endregion + #region Test Classes public class EmptyBenchmark { } @@ -57,6 +341,7 @@ public class EmptyBenchmark2 { } + [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, invocationCount: 1, id: "QuickJob")] public class NotEmptyBenchmark { [Benchmark] @@ -69,5 +354,6 @@ public void Benchmark() } } } + #endregion } } \ No newline at end of file From ffdf934e097afecd0752f7fffaabd226284ddbb7 Mon Sep 17 00:00:00 2001 From: Avishai Dotan <108017307+AvishaiDotan@users.noreply.github.com> Date: Thu, 1 May 2025 21:50:01 +0300 Subject: [PATCH 8/8] fix benchmarswitcher tests error --- src/BenchmarkDotNet/Running/TypeFilter.cs | 8 ++++---- .../BenchmarkSwitcherTest.cs | 11 +++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/BenchmarkDotNet/Running/TypeFilter.cs b/src/BenchmarkDotNet/Running/TypeFilter.cs index 741ec3fe3f..4d2188844f 100644 --- a/src/BenchmarkDotNet/Running/TypeFilter.cs +++ b/src/BenchmarkDotNet/Running/TypeFilter.cs @@ -26,7 +26,6 @@ public static (bool allTypesValid, IReadOnlyList runnable) GetTypesWithRun { logger.WriteLineError($"No [Benchmark] attribute found on '{type.Name}' benchmark case."); } - return (false, Array.Empty()); } else if (assemblies.Any()) { @@ -34,11 +33,12 @@ public static (bool allTypesValid, IReadOnlyList runnable) GetTypesWithRun { logger.WriteLineError($"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly."); } - return (false, Array.Empty()); } - logger.WriteLineError("No benchmarks were found."); + else + { + logger.WriteLineError("No benchmarks were found."); + } return (false, Array.Empty()); - } foreach (var type in types) diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs index e9a95fce81..622aee3ffd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs @@ -66,7 +66,7 @@ public void WhenInvalidTypeIsProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAre .Run(new[] { "--filter", "*" }, config); Assert.Empty(summaries); - Assert.Contains("Type BenchmarkDotNet.IntegrationTests.ClassC is invalid.", logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(ClassC)), logger.GetLog()); } [Fact] @@ -80,7 +80,7 @@ public void WhenNoTypesAreProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAreExe .Run(new[] { "--filter", "*" }, config); Assert.Empty(summaries); - Assert.Contains("No benchmarks to choose from. Make sure you provided public non-sealed non-static types with public [Benchmark] methods.", logger.GetLog()); + Assert.Contains("No benchmarks were found.", logger.GetLog()); } [Fact] @@ -369,6 +369,11 @@ public IReadOnlyList AskUser(IReadOnlyList allTypes, ILogger logger) return returnValue; } } + + private string GetValidationErrorForType(Type type) + { + return $"No [Benchmark] attribute found on '{type.Name}' benchmark case."; + } } } @@ -424,6 +429,8 @@ public override void ExportToLog(Summary summary, ILogger logger) exported = true; } } + + } namespace BenchmarkDotNet.NOTIntegrationTests