diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 4f520da38d..e29cc22f28 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -19,7 +19,6 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Mathematics; -using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.Parameters; @@ -74,9 +73,6 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) if (validationErrors.Any(validationError => validationError.IsCritical)) return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath, validationErrors.ToImmutableArray()) }; - if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any())) - return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) }; - eventProcessor.OnEndValidationStage(); int totalBenchmarkCount = supportedBenchmarks.Sum(benchmarkInfo => benchmarkInfo.BenchmarksCases.Length); @@ -579,20 +575,47 @@ private static void LogTotalTime(ILogger logger, TimeSpan time, int executedBenc private static (BenchmarkRunInfo[], List) GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, IResolver resolver) { List validationErrors = new (); + List runInfos = new (benchmarkRunInfos.Length); + + if (benchmarkRunInfos.Length == 0) + { + validationErrors.Add(new ValidationError(true, $"No benchmarks were found.")); + return (Array.Empty(), validationErrors); + } + + foreach (var benchmarkRunInfo in benchmarkRunInfos) + { + if (benchmarkRunInfo.BenchmarksCases.Length == 0) + { + validationErrors.Add(new ValidationError(true, $"No [Benchmark] attribute found on '{benchmarkRunInfo.Type.Name}' benchmark case.")); + continue; + } - var runInfos = benchmarkRunInfos.Select(info => new BenchmarkRunInfo( - info.BenchmarksCases.Where(benchmark => + var validBenchmarks = benchmarkRunInfo.BenchmarksCases + .Where(benchmark => { - var errors = benchmark.GetToolchain().Validate(benchmark, resolver).ToArray(); + + var errors = benchmark.GetToolchain() + .Validate(benchmark, resolver) + .ToArray(); + validationErrors.AddRange(errors); - return !errors.Any(); - }).ToArray(), - info.Type, - info.Config)) - .Where(infos => infos.BenchmarksCases.Any()) - .ToArray(); - - return (runInfos, validationErrors); + + return errors.Length == 0; + }) + .ToArray(); + + runInfos.Add( + new BenchmarkRunInfo( + validBenchmarks, + benchmarkRunInfo.Type, + benchmarkRunInfo.Config + + )); + + + } + return (runInfos.ToArray(), validationErrors); } private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRunInfos) @@ -665,6 +688,9 @@ void AddLogger(ILogger logger) foreach (var logger in benchmarkRunInfo.Config.GetLoggers()) AddLogger(logger); + if (benchmarkRunInfos.Length == 0) + AddLogger(new ConsoleLogger()); + AddLogger(streamLogger); return new CompositeLogger(loggers.Values.ToImmutableHashSet()); diff --git a/src/BenchmarkDotNet/Running/TypeFilter.cs b/src/BenchmarkDotNet/Running/TypeFilter.cs index 4e58ebdeaf..4d2188844f 100644 --- a/src/BenchmarkDotNet/Running/TypeFilter.cs +++ b/src/BenchmarkDotNet/Running/TypeFilter.cs @@ -15,6 +15,32 @@ public static (bool allTypesValid, IReadOnlyList runnable) GetTypesWithRun { var validRunnableTypes = new List(); + bool hasRunnableTypeBenchmarks = types.Any(type => type.ContainsRunnableBenchmarks()); + bool hasRunnableAssemblyBenchmarks = assemblies.Any(assembly => GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks()).Length > 0); + + if (!hasRunnableTypeBenchmarks && !hasRunnableAssemblyBenchmarks) + { + if (types.Any()) + { + foreach (var type in types) + { + logger.WriteLineError($"No [Benchmark] attribute found on '{type.Name}' benchmark case."); + } + } + else if (assemblies.Any()) + { + foreach (var assembly in assemblies) + { + logger.WriteLineError($"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly."); + } + } + else + { + logger.WriteLineError("No benchmarks were found."); + } + return (false, Array.Empty()); + } + foreach (var type in types) { if (type.ContainsRunnableBenchmarks()) diff --git a/src/BenchmarkDotNet/Running/WakeLock.cs b/src/BenchmarkDotNet/Running/WakeLock.cs index a748f24626..124fbd5e22 100644 --- a/src/BenchmarkDotNet/Running/WakeLock.cs +++ b/src/BenchmarkDotNet/Running/WakeLock.cs @@ -11,7 +11,7 @@ namespace BenchmarkDotNet.Running; internal partial class WakeLock { public static WakeLockType GetWakeLockType(BenchmarkRunInfo[] benchmarkRunInfos) => - benchmarkRunInfos.Select(static i => i.Config.WakeLock).Max(); + benchmarkRunInfos.Length == 0 ? WakeLockType.None : benchmarkRunInfos.Select(static i => i.Config.WakeLock).Max(); private static readonly bool OsVersionIsSupported = // Must be windows 7 or greater diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs index e9a95fce81..622aee3ffd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs @@ -66,7 +66,7 @@ public void WhenInvalidTypeIsProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAre .Run(new[] { "--filter", "*" }, config); Assert.Empty(summaries); - Assert.Contains("Type BenchmarkDotNet.IntegrationTests.ClassC is invalid.", logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(ClassC)), logger.GetLog()); } [Fact] @@ -80,7 +80,7 @@ public void WhenNoTypesAreProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAreExe .Run(new[] { "--filter", "*" }, config); Assert.Empty(summaries); - Assert.Contains("No benchmarks to choose from. Make sure you provided public non-sealed non-static types with public [Benchmark] methods.", logger.GetLog()); + Assert.Contains("No benchmarks were found.", logger.GetLog()); } [Fact] @@ -369,6 +369,11 @@ public IReadOnlyList AskUser(IReadOnlyList allTypes, ILogger logger) return returnValue; } } + + private string GetValidationErrorForType(Type type) + { + return $"No [Benchmark] attribute found on '{type.Name}' benchmark case."; + } } } @@ -424,6 +429,8 @@ public override void ExportToLog(Summary summary, ILogger logger) exported = true; } } + + } namespace BenchmarkDotNet.NOTIntegrationTests diff --git a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs index 4cede6835b..f9272edd2d 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs @@ -23,8 +23,9 @@ public class EventProcessorTests public void WhenUsingEventProcessorAndNoBenchmarks() { var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassEmpty) }); - Assert.Single(events); + Assert.Equal(2, events.Count); Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType); } [Fact] diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs new file mode 100644 index 0000000000..05ce92b2cf --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -0,0 +1,359 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Running; +using Xunit; +using System.Reflection; +using System.Reflection.Emit; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Tests.Running +{ + public class RunningEmptyBenchmarkTests + { + #region BenchmarkRunner Methods Overview + /* + * Available BenchmarkRunner.Run methods: + * 1. Generic Type: + * - BenchmarkRunner.Run(IConfig? config = null, string[]? args = null) + * 2. Type-based: + * - BenchmarkRunner.Run(Type type, IConfig? config = null, string[]? args = null) + * - BenchmarkRunner.Run(Type[] types, IConfig? config = null, string[]? args = null) + * - BenchmarkRunner.Run(Type type, MethodInfo[] methods, IConfig? config = null) + * 3. Assembly-based: + * - BenchmarkRunner.Run(Assembly assembly, IConfig? config = null, string[]? args = null) + * 4. BenchmarkRunInfo-based: + * - BenchmarkRunner.Run(BenchmarkRunInfo benchmarkRunInfo) + * - BenchmarkRunner.Run(BenchmarkRunInfo[] benchmarkRunInfos) + * 5. Deprecated methods: + * - BenchmarkRunner.RunUrl(string url, IConfig? config = null) + * - BenchmarkRunner.RunSource(string source, IConfig? config = null) + */ + #endregion + #region Generic Type Tests + /// + /// Tests for BenchmarkRunner.Run method + /// + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void GenericTypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(config, args); + if (args == null) + { + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + } + + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void GenericTypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(config, args); + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region Type-based Tests + /// + /// Tests for BenchmarkRunner.Run(Type) method + /// + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void TypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + + var summary = BenchmarkRunner.Run(typeof(EmptyBenchmark), config, args); + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(typeof(NotEmptyBenchmark), config, args); + Assert.False(summaries.HasCriticalValidationErrors); + Assert.DoesNotContain(summaries.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + + /// + /// Tests for BenchmarkRunner.Run(Type[]) method + /// + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypesWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(EmptyBenchmark), typeof(EmptyBenchmark2) }, config, args); + if (args != null) + { + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + else + { + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark2))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + + + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypesWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(NotEmptyBenchmark) }, config, args); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region BenchmarkRunInfo Tests + /// + /// Tests for BenchmarkRunner.Run(BenchmarkRunInfo) method + /// + [Fact] + public void BenchmarkRunInfoWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute() + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config)); + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Fact] + public void BenchmarkRunInfoWithBenchmarkAttribute_RunsSuccessfully() + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config)); + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + + /// + /// Tests for BenchmarkRunner.Run(BenchmarkRunInfo[]) method + /// + [Fact] + public void BenchmarkRunInfosWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute() + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { + BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config), + BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark2), config) + }); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark2))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + + [Fact] + public void BenchmarkRunInfosWithBenchmarkAttribute_RunsSuccessfully() + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config) }); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region Mixed Types Tests + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void MixedTypes_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(EmptyBenchmark), typeof(NotEmptyBenchmark) }, config, args); + if (args != null) + { + Assert.Contains(GetExpandedValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + else + { + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + } + #endregion + #region Assembly Tests + // In this tests there is no config and logger because the logger is initiated at CreateCompositeLogger when the BenchmarkRunInfo[] is empty + // those cannot be inserted using config + [Theory] + + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void AssemblyWithoutBenchmarks_ThrowsValidationError_WhenNoBenchmarksFound(string[]? args) + { + + // Create a mock assembly with no benchmark types + var assemblyName = new AssemblyName("MockAssembly"); + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule("MockModule"); + // Create a simple type in the assembly (no benchmarks) + var typeBuilder = moduleBuilder.DefineType("MockType", TypeAttributes.Public); + typeBuilder.CreateType(); + + Summary[] summaries = null; + if (args != null) + { + GetConfigWithLogger(out var logger, out var config); + summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + Assert.Contains(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); + } + else + { + summaries = BenchmarkRunner.Run(assemblyBuilder, null, args); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); + } + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void AssemblyWithBenchmarks_RunsSuccessfully_WhenBenchmarkAttributePresent(string[]? args) + { + // Skip test on .NET Framework 4.6.2 + if (RuntimeInformation.FrameworkDescription.Contains(".NET Framework 4")) + return; + + // Create a mock assembly with benchmark types + var assemblyName = new AssemblyName("MockAssemblyWithBenchmarks"); + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule("MockModule"); + + // Create a benchmark type + var benchmarkTypeBuilder = moduleBuilder.DefineType("MockBenchmark", TypeAttributes.Public); + var benchmarkMethod = benchmarkTypeBuilder.DefineMethod("Benchmark", MethodAttributes.Public, typeof(void), Type.EmptyTypes); + + // Generate method body + var ilGenerator = benchmarkMethod.GetILGenerator(); + ilGenerator.Emit(OpCodes.Ret); // Just return from the method + + var benchmarkAttributeCtor = typeof(BenchmarkAttribute).GetConstructor(new[] { typeof(int), typeof(string) }); + if (benchmarkAttributeCtor == null) + throw new InvalidOperationException("Could not find BenchmarkAttribute constructor"); + benchmarkMethod.SetCustomAttribute(new CustomAttributeBuilder( + benchmarkAttributeCtor, + new object[] { 0, "" })); + benchmarkTypeBuilder.CreateType(); + + Summary[] summaries = null; + if (args != null) + { + GetConfigWithLogger(out var logger, out var config); + summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + Assert.DoesNotContain(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); + } + else + { + summaries = BenchmarkRunner.Run(assemblyBuilder); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); + } + } + #endregion + #region Helper Methods + private string GetValidationErrorForType(Type type) + { + return $"No [Benchmark] attribute found on '{type.Name}' benchmark case."; + } + + private string GetAssemblylValidationError(Assembly assembly) + { + return $"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly."; + } + + private string GetExpandedValidationErrorForType(Type type) + { + return $"Type {type} is invalid. Only public, non-generic (closed generic types with public parameterless ctors are supported), non-abstract, non-sealed, non-static types with public instance [Benchmark] method(s) are supported."; + } + + private string GetGeneralValidationError() + { + return $"No benchmarks were found."; + } + + private void GetConfigWithLogger(out AccumulationLogger logger, out ManualConfig manualConfig) + { + logger = new AccumulationLogger(); + manualConfig = ManualConfig.CreateEmpty() + .AddLogger(logger) + .AddColumnProvider(DefaultColumnProviders.Instance); + } + + #endregion + #region Test Classes + public class EmptyBenchmark + { + } + + public class EmptyBenchmark2 + { + } + + [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, invocationCount: 1, id: "QuickJob")] + public class NotEmptyBenchmark + { + [Benchmark] + public void Benchmark() + { + var sum = 0; + for (int i = 0; i < 1; i++) + { + sum += i; + } + } + } + #endregion + } +} \ No newline at end of file