From 991de5780d1017d2596ffd8b1421a594600f9779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Fri, 1 May 2026 17:47:24 +0200 Subject: [PATCH 1/2] feat: implement baseline ratio computation and unit normalization in benchmark reporting --- Pipeline/BenchmarkReport.cs | 102 ++++++++++++++++-- .../BenchmarkReportHelpersTests.cs | 89 +++++++++++++++ Tests/Build.Tests/BuildBodyTests.cs | 92 ++++++++++++++++ 3 files changed, 276 insertions(+), 7 deletions(-) diff --git a/Pipeline/BenchmarkReport.cs b/Pipeline/BenchmarkReport.cs index e4370ab6..24f12601 100644 --- a/Pipeline/BenchmarkReport.cs +++ b/Pipeline/BenchmarkReport.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Text; @@ -11,8 +12,6 @@ namespace Build; /// internal static class BenchmarkReport { - public static readonly string[] DefaultColumnsToRemove = ["RatioSD", "Gen0", "Gen1", "Gen2",]; - public static string BuildBody(IReadOnlyList files, string[] columnsToRemove) { StringBuilder sb = new(); @@ -121,10 +120,22 @@ internal static void FlushTableBuffer(StringBuilder sb, List tableBuff string commonPrefix = FindCommonRowPrefix(tableBuffer); + TableRow headerRow = tableBuffer[0].RowIndex == 0 ? tableBuffer[0] : null; + int allocatedIndex = headerRow != null ? FindColumnIndex(headerRow.Tokens, "Allocated") : -1; + int ratioIndex = headerRow != null ? FindColumnIndex(headerRow.Tokens, "Ratio") : -1; + int allocRatioIndex = headerRow != null ? FindColumnIndex(headerRow.Tokens, "Alloc Ratio") : -1; + foreach (TableRow row in tableBuffer) { if (row.RowIndex >= BenchmarkTableParser.DataRowStartIndex && row.Tokens.Length == 0) { + if (headerRow != null && headerRow.Tokens.Length > 0) + { + string[] empty = new string[headerRow.Tokens.Length]; + Array.Fill(empty, string.Empty); + sb.AppendLine(JoinTokens(empty)); + } + continue; } @@ -140,14 +151,18 @@ internal static void FlushTableBuffer(StringBuilder sb, List tableBuff string key = string.Join("|", row.Tokens.Take(meanIndex)); if (baselineRows.TryGetValue(key, out string[] baselineTokens) && baselineTokens.Length > 0) { - string[] modifiedBaseline = new string[baselineTokens.Length]; - for (int i = 0; i < baselineTokens.Length; i++) + string[] modifiedBaseline = (string[])baselineTokens.Clone(); + RecomputeBaselineRatio(modifiedBaseline, row.Tokens, meanIndex, ratioIndex); + RecomputeBaselineRatio(modifiedBaseline, row.Tokens, allocatedIndex, allocRatioIndex); + + string[] italicized = new string[modifiedBaseline.Length]; + for (int i = 0; i < modifiedBaseline.Length; i++) { - string content = i == 0 ? "baseline*" : baselineTokens[i]; - modifiedBaseline[i] = string.IsNullOrWhiteSpace(content) ? content : $"_{content}_"; + string content = i == 0 ? "baseline*" : modifiedBaseline[i]; + italicized[i] = string.IsNullOrWhiteSpace(content) ? content : $"_{content}_"; } - sb.AppendLine(JoinTokens(modifiedBaseline)); + sb.AppendLine(JoinTokens(italicized)); anyBaselineInjected = true; } } @@ -165,6 +180,30 @@ internal static void FlushTableBuffer(StringBuilder sb, List tableBuff tableBuffer.Clear(); } + internal static void RecomputeBaselineRatio(string[] baselineTokens, string[] mockolateTokens, + int valueIndex, int ratioIndex) + { + if (valueIndex <= 0 || ratioIndex <= 0) + { + return; + } + + if (valueIndex >= baselineTokens.Length || valueIndex >= mockolateTokens.Length + || ratioIndex >= baselineTokens.Length) + { + return; + } + + if (!TryParseQuantity(baselineTokens[valueIndex], out double baselineValue) + || !TryParseQuantity(mockolateTokens[valueIndex], out double mockolateValue) + || mockolateValue <= 0) + { + return; + } + + baselineTokens[ratioIndex] = (baselineValue / mockolateValue).ToString("F2", CultureInfo.InvariantCulture); + } + internal static string[] ApplyHeaderAndPrefixStripping(TableRow row, string commonPrefix) { if (commonPrefix == null || row.Tokens.Length == 0) @@ -241,6 +280,55 @@ internal static string JoinTokens(string[] tokens) return sb.ToString(); } + internal static int FindColumnIndex(string[] tokens, string columnName) => + Array.FindIndex(tokens, t => string.Equals(t, columnName, StringComparison.OrdinalIgnoreCase)); + + internal static bool TryParseQuantity(string value, out double normalized) + { + normalized = 0; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + string trimmed = value.Trim(); + int spaceIdx = trimmed.LastIndexOf(' '); + string numberPart = spaceIdx > 0 ? trimmed.Substring(0, spaceIdx).Trim() : trimmed; + string unitPart = spaceIdx > 0 ? trimmed.Substring(spaceIdx + 1).Trim() : string.Empty; + + if (!double.TryParse(numberPart, + NumberStyles.Float | NumberStyles.AllowThousands, + CultureInfo.InvariantCulture, + out double number)) + { + return false; + } + + double multiplier = unitPart switch + { + "" => 1, + "ps" => 0.001, + "ns" => 1, + "μs" => 1000, + "us" => 1000, + "ms" => 1_000_000, + "s" => 1_000_000_000, + "B" => 1, + "KB" => 1024, + "MB" => 1024d * 1024, + "GB" => 1024d * 1024 * 1024, + _ => double.NaN, + }; + + if (double.IsNaN(multiplier)) + { + return false; + } + + normalized = number * multiplier; + return true; + } + internal static int[] DetermineDroppedColumnIndices(string headerLine, string[] columnsToRemove) { string[] tokens = SplitTableRow(headerLine); diff --git a/Tests/Build.Tests/BenchmarkReportHelpersTests.cs b/Tests/Build.Tests/BenchmarkReportHelpersTests.cs index ee39639b..99a71150 100644 --- a/Tests/Build.Tests/BenchmarkReportHelpersTests.cs +++ b/Tests/Build.Tests/BenchmarkReportHelpersTests.cs @@ -52,6 +52,22 @@ public async Task DetermineDroppedColumnIndices_WhenNoColumnsMatch_ShouldReturnE await That(indices).IsEqualTo(Array.Empty()); } + [Fact] + public async Task FindColumnIndex_ShouldMatchCaseInsensitively() + { + int index = BenchmarkReport.FindColumnIndex(["Method", "Mean", "Alloc Ratio",], "alloc ratio"); + + await That(index).IsEqualTo(2); + } + + [Fact] + public async Task FindColumnIndex_WhenColumnNotFound_ShouldReturnMinusOne() + { + int index = BenchmarkReport.FindColumnIndex(["Method", "Mean",], "Ratio"); + + await That(index).IsEqualTo(-1); + } + [Fact] public async Task FindCommonRowPrefix_ShouldIgnoreHeaderAndSeparatorRows() { @@ -141,6 +157,39 @@ public async Task FindCommonRowPrefix_WhenDataRowsDisagree_ShouldReturnNull() await That(prefix).IsNull(); } + [Fact] + public async Task RecomputeBaselineRatio_ShouldOverwriteRatioWithBaselineDividedByMockolate() + { + string[] baseline = ["Foo_Mockolate", "206.44 ns", "1.00", "1048 B", "1.00",]; + string[] mockolate = ["Foo_Mockolate", "216.85 ns", "1.00", "1048 B", "1.00",]; + + BenchmarkReport.RecomputeBaselineRatio(baseline, mockolate, 1, 2); + + await That(baseline[2]).IsEqualTo("0.95"); + } + + [Fact] + public async Task RecomputeBaselineRatio_WhenRatioIndexIsMissing_ShouldNotMutate() + { + string[] baseline = ["Foo_Mockolate", "206.44 ns",]; + string[] mockolate = ["Foo_Mockolate", "216.85 ns",]; + + BenchmarkReport.RecomputeBaselineRatio(baseline, mockolate, 1, -1); + + await That(baseline).IsEqualTo(["Foo_Mockolate", "206.44 ns",]); + } + + [Fact] + public async Task RecomputeBaselineRatio_WhenValueCannotBeParsed_ShouldNotMutate() + { + string[] baseline = ["Foo_Mockolate", "junk", "1.00",]; + string[] mockolate = ["Foo_Mockolate", "216.85 ns", "1.00",]; + + BenchmarkReport.RecomputeBaselineRatio(baseline, mockolate, 1, 2); + + await That(baseline[2]).IsEqualTo("1.00"); + } + [Fact] public async Task RemoveColumns_ShouldDropTheGivenIndicesAndPreserveTheRest() { @@ -176,4 +225,44 @@ public async Task SplitTableRow_WhenLineHasNoPipe_ShouldReturnEmptyArray() await That(tokens).IsEqualTo(Array.Empty()); } + + [Theory] + [InlineData("1048 B", 1048.0)] + [InlineData("1 KB", 1024.0)] + [InlineData("2 MB", 2.0 * 1024 * 1024)] + [InlineData("3 GB", 3.0 * 1024 * 1024 * 1024)] + public async Task TryParseQuantity_ShouldNormalizeMemoryUnitsToBytes(string value, double expected) + { + bool ok = BenchmarkReport.TryParseQuantity(value, out double actual); + + await That(ok).IsTrue(); + await That(actual).IsEqualTo(expected).Within(0.0001); + } + + [Theory] + [InlineData("206.44 ns", 206.44)] + [InlineData("1,377.70 ns", 1377.70)] + [InlineData("1.00 μs", 1000.0)] + [InlineData("1.00 us", 1000.0)] + [InlineData("2 ms", 2_000_000.0)] + [InlineData("3 s", 3_000_000_000.0)] + [InlineData("500 ps", 0.5)] + public async Task TryParseQuantity_ShouldNormalizeTimeUnitsToNanoseconds(string value, double expected) + { + bool ok = BenchmarkReport.TryParseQuantity(value, out double actual); + + await That(ok).IsTrue(); + await That(actual).IsEqualTo(expected).Within(0.0001); + } + + [Theory] + [InlineData("")] + [InlineData("not a number")] + [InlineData("100 weird")] + public async Task TryParseQuantity_WhenInputIsInvalid_ShouldReturnFalse(string value) + { + bool ok = BenchmarkReport.TryParseQuantity(value, out _); + + await That(ok).IsFalse(); + } } diff --git a/Tests/Build.Tests/BuildBodyTests.cs b/Tests/Build.Tests/BuildBodyTests.cs index 7b3cd611..6549fe22 100644 --- a/Tests/Build.Tests/BuildBodyTests.cs +++ b/Tests/Build.Tests/BuildBodyTests.cs @@ -148,6 +148,31 @@ public async Task BuildBody_WhenTableHasEmptyRowSeparatingParameterGroups_Should await That(body).DoesNotContain($"{Environment.NewLine}|{Environment.NewLine}"); } + [Fact] + public async Task BuildBody_WhenTableHasEmptyRowSeparatingParameterGroups_ShouldEmitEmptyRowMatchingHeaderColumnCount() + { + string[] report = + [ + "| Method | N | Mean | Allocated |", + "|--------|---|-----:|----------:|", + "| Indexer_Mockolate | 1 | 100 ns | 1 KB |", + "| Indexer_Moq | 1 | 200 ns | 2 KB |", + "| | | | |", + "| Indexer_Mockolate | 10 | 1000 ns | 10 KB |", + "| Indexer_Moq | 10 | 2000 ns | 20 KB |", + ]; + BenchmarkReportFile file = new(report, null); + + string body = BenchmarkReport.BuildBody([file,], _columnsToRemove); + + string[] lines = body.Split(Environment.NewLine); + int firstGroupLast = Array.IndexOf(lines, "| Moq | 1 | 200 ns | 2 KB |"); + int secondGroupFirst = Array.IndexOf(lines, "| **Mockolate** | **10** | **1000 ns** | **10 KB** |"); + await That(firstGroupLast).IsGreaterThan(-1); + await That(secondGroupFirst).IsGreaterThan(firstGroupLast); + await That(lines[firstGroupLast + 1]).IsEqualTo("| | | | |"); + } + [Fact] public async Task BuildBody_WhenTableHasEmptyRowSeparatingParameterGroups_ShouldStillStripCommonPrefix() { @@ -195,6 +220,73 @@ await That(body).Contains( "`baseline*` rows show the corresponding Mockolate benchmark"); } + [Fact] + public async Task BuildBody_WithBaseline_ShouldNormalizeUnitDifferencesWhenComputingRatio() + { + string[] report = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 500 ns | 1.00 | 1 KB | 1.00 |", + ]; + string[] baseline = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 1.00 μs | 1.00 | 2048 B | 1.00 |", + ]; + BenchmarkReportFile file = new(report, baseline); + + string body = BenchmarkReport.BuildBody([file,], _columnsToRemove); + + await That(body).Contains("| _baseline*_ | _1.00 μs_ | _2.00_ | _2048 B_ | _2.00_ |"); + } + + [Fact] + public async Task BuildBody_WithBaseline_ShouldRecomputeAllocRatioRelativeToCurrentMockolate() + { + string[] report = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 100 ns | 1.00 | 2 KB | 1.00 |", + ]; + string[] baseline = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 100 ns | 1.00 | 1024 B | 1.00 |", + ]; + BenchmarkReportFile file = new(report, baseline); + + string body = BenchmarkReport.BuildBody([file,], _columnsToRemove); + + await That(body).Contains("| _baseline*_ | _100 ns_ | _1.00_ | _1024 B_ | _0.50_ |"); + } + + [Fact] + public async Task BuildBody_WithBaseline_ShouldRecomputeRatioRelativeToCurrentMockolate() + { + string[] report = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 216.85 ns | 1.00 | 1048 B | 1.00 |", + "| Foo_Moq | 1377.70 ns | 6.36 | 2096 B | 2.00 |", + ]; + string[] baseline = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 206.44 ns | 1.00 | 1048 B | 1.00 |", + ]; + BenchmarkReportFile file = new(report, baseline); + + string body = BenchmarkReport.BuildBody([file,], _columnsToRemove); + + await That(body).Contains("| _baseline*_ | _206.44 ns_ | _0.95_ | _1048 B_ | _1.00_ |"); + } + [Fact] public async Task BuildBody_WithoutBaseline_ShouldNotEmitBaselineNote() { From 36f565e4fd24068187ea37678566b84d37a87db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Breu=C3=9F?= Date: Fri, 1 May 2026 17:47:24 +0200 Subject: [PATCH 2/2] feat: implement baseline ratio computation and unit normalization in benchmark reporting --- Pipeline/BenchmarkReport.cs | 102 ++++++++++++++++-- .../BenchmarkReportHelpersTests.cs | 89 +++++++++++++++ Tests/Build.Tests/BuildBodyTests.cs | 92 ++++++++++++++++ 3 files changed, 276 insertions(+), 7 deletions(-) diff --git a/Pipeline/BenchmarkReport.cs b/Pipeline/BenchmarkReport.cs index e4370ab6..24f12601 100644 --- a/Pipeline/BenchmarkReport.cs +++ b/Pipeline/BenchmarkReport.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Text; @@ -11,8 +12,6 @@ namespace Build; /// internal static class BenchmarkReport { - public static readonly string[] DefaultColumnsToRemove = ["RatioSD", "Gen0", "Gen1", "Gen2",]; - public static string BuildBody(IReadOnlyList files, string[] columnsToRemove) { StringBuilder sb = new(); @@ -121,10 +120,22 @@ internal static void FlushTableBuffer(StringBuilder sb, List tableBuff string commonPrefix = FindCommonRowPrefix(tableBuffer); + TableRow headerRow = tableBuffer[0].RowIndex == 0 ? tableBuffer[0] : null; + int allocatedIndex = headerRow != null ? FindColumnIndex(headerRow.Tokens, "Allocated") : -1; + int ratioIndex = headerRow != null ? FindColumnIndex(headerRow.Tokens, "Ratio") : -1; + int allocRatioIndex = headerRow != null ? FindColumnIndex(headerRow.Tokens, "Alloc Ratio") : -1; + foreach (TableRow row in tableBuffer) { if (row.RowIndex >= BenchmarkTableParser.DataRowStartIndex && row.Tokens.Length == 0) { + if (headerRow != null && headerRow.Tokens.Length > 0) + { + string[] empty = new string[headerRow.Tokens.Length]; + Array.Fill(empty, string.Empty); + sb.AppendLine(JoinTokens(empty)); + } + continue; } @@ -140,14 +151,18 @@ internal static void FlushTableBuffer(StringBuilder sb, List tableBuff string key = string.Join("|", row.Tokens.Take(meanIndex)); if (baselineRows.TryGetValue(key, out string[] baselineTokens) && baselineTokens.Length > 0) { - string[] modifiedBaseline = new string[baselineTokens.Length]; - for (int i = 0; i < baselineTokens.Length; i++) + string[] modifiedBaseline = (string[])baselineTokens.Clone(); + RecomputeBaselineRatio(modifiedBaseline, row.Tokens, meanIndex, ratioIndex); + RecomputeBaselineRatio(modifiedBaseline, row.Tokens, allocatedIndex, allocRatioIndex); + + string[] italicized = new string[modifiedBaseline.Length]; + for (int i = 0; i < modifiedBaseline.Length; i++) { - string content = i == 0 ? "baseline*" : baselineTokens[i]; - modifiedBaseline[i] = string.IsNullOrWhiteSpace(content) ? content : $"_{content}_"; + string content = i == 0 ? "baseline*" : modifiedBaseline[i]; + italicized[i] = string.IsNullOrWhiteSpace(content) ? content : $"_{content}_"; } - sb.AppendLine(JoinTokens(modifiedBaseline)); + sb.AppendLine(JoinTokens(italicized)); anyBaselineInjected = true; } } @@ -165,6 +180,30 @@ internal static void FlushTableBuffer(StringBuilder sb, List tableBuff tableBuffer.Clear(); } + internal static void RecomputeBaselineRatio(string[] baselineTokens, string[] mockolateTokens, + int valueIndex, int ratioIndex) + { + if (valueIndex <= 0 || ratioIndex <= 0) + { + return; + } + + if (valueIndex >= baselineTokens.Length || valueIndex >= mockolateTokens.Length + || ratioIndex >= baselineTokens.Length) + { + return; + } + + if (!TryParseQuantity(baselineTokens[valueIndex], out double baselineValue) + || !TryParseQuantity(mockolateTokens[valueIndex], out double mockolateValue) + || mockolateValue <= 0) + { + return; + } + + baselineTokens[ratioIndex] = (baselineValue / mockolateValue).ToString("F2", CultureInfo.InvariantCulture); + } + internal static string[] ApplyHeaderAndPrefixStripping(TableRow row, string commonPrefix) { if (commonPrefix == null || row.Tokens.Length == 0) @@ -241,6 +280,55 @@ internal static string JoinTokens(string[] tokens) return sb.ToString(); } + internal static int FindColumnIndex(string[] tokens, string columnName) => + Array.FindIndex(tokens, t => string.Equals(t, columnName, StringComparison.OrdinalIgnoreCase)); + + internal static bool TryParseQuantity(string value, out double normalized) + { + normalized = 0; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + string trimmed = value.Trim(); + int spaceIdx = trimmed.LastIndexOf(' '); + string numberPart = spaceIdx > 0 ? trimmed.Substring(0, spaceIdx).Trim() : trimmed; + string unitPart = spaceIdx > 0 ? trimmed.Substring(spaceIdx + 1).Trim() : string.Empty; + + if (!double.TryParse(numberPart, + NumberStyles.Float | NumberStyles.AllowThousands, + CultureInfo.InvariantCulture, + out double number)) + { + return false; + } + + double multiplier = unitPart switch + { + "" => 1, + "ps" => 0.001, + "ns" => 1, + "μs" => 1000, + "us" => 1000, + "ms" => 1_000_000, + "s" => 1_000_000_000, + "B" => 1, + "KB" => 1024, + "MB" => 1024d * 1024, + "GB" => 1024d * 1024 * 1024, + _ => double.NaN, + }; + + if (double.IsNaN(multiplier)) + { + return false; + } + + normalized = number * multiplier; + return true; + } + internal static int[] DetermineDroppedColumnIndices(string headerLine, string[] columnsToRemove) { string[] tokens = SplitTableRow(headerLine); diff --git a/Tests/Build.Tests/BenchmarkReportHelpersTests.cs b/Tests/Build.Tests/BenchmarkReportHelpersTests.cs index ee39639b..99a71150 100644 --- a/Tests/Build.Tests/BenchmarkReportHelpersTests.cs +++ b/Tests/Build.Tests/BenchmarkReportHelpersTests.cs @@ -52,6 +52,22 @@ public async Task DetermineDroppedColumnIndices_WhenNoColumnsMatch_ShouldReturnE await That(indices).IsEqualTo(Array.Empty()); } + [Fact] + public async Task FindColumnIndex_ShouldMatchCaseInsensitively() + { + int index = BenchmarkReport.FindColumnIndex(["Method", "Mean", "Alloc Ratio",], "alloc ratio"); + + await That(index).IsEqualTo(2); + } + + [Fact] + public async Task FindColumnIndex_WhenColumnNotFound_ShouldReturnMinusOne() + { + int index = BenchmarkReport.FindColumnIndex(["Method", "Mean",], "Ratio"); + + await That(index).IsEqualTo(-1); + } + [Fact] public async Task FindCommonRowPrefix_ShouldIgnoreHeaderAndSeparatorRows() { @@ -141,6 +157,39 @@ public async Task FindCommonRowPrefix_WhenDataRowsDisagree_ShouldReturnNull() await That(prefix).IsNull(); } + [Fact] + public async Task RecomputeBaselineRatio_ShouldOverwriteRatioWithBaselineDividedByMockolate() + { + string[] baseline = ["Foo_Mockolate", "206.44 ns", "1.00", "1048 B", "1.00",]; + string[] mockolate = ["Foo_Mockolate", "216.85 ns", "1.00", "1048 B", "1.00",]; + + BenchmarkReport.RecomputeBaselineRatio(baseline, mockolate, 1, 2); + + await That(baseline[2]).IsEqualTo("0.95"); + } + + [Fact] + public async Task RecomputeBaselineRatio_WhenRatioIndexIsMissing_ShouldNotMutate() + { + string[] baseline = ["Foo_Mockolate", "206.44 ns",]; + string[] mockolate = ["Foo_Mockolate", "216.85 ns",]; + + BenchmarkReport.RecomputeBaselineRatio(baseline, mockolate, 1, -1); + + await That(baseline).IsEqualTo(["Foo_Mockolate", "206.44 ns",]); + } + + [Fact] + public async Task RecomputeBaselineRatio_WhenValueCannotBeParsed_ShouldNotMutate() + { + string[] baseline = ["Foo_Mockolate", "junk", "1.00",]; + string[] mockolate = ["Foo_Mockolate", "216.85 ns", "1.00",]; + + BenchmarkReport.RecomputeBaselineRatio(baseline, mockolate, 1, 2); + + await That(baseline[2]).IsEqualTo("1.00"); + } + [Fact] public async Task RemoveColumns_ShouldDropTheGivenIndicesAndPreserveTheRest() { @@ -176,4 +225,44 @@ public async Task SplitTableRow_WhenLineHasNoPipe_ShouldReturnEmptyArray() await That(tokens).IsEqualTo(Array.Empty()); } + + [Theory] + [InlineData("1048 B", 1048.0)] + [InlineData("1 KB", 1024.0)] + [InlineData("2 MB", 2.0 * 1024 * 1024)] + [InlineData("3 GB", 3.0 * 1024 * 1024 * 1024)] + public async Task TryParseQuantity_ShouldNormalizeMemoryUnitsToBytes(string value, double expected) + { + bool ok = BenchmarkReport.TryParseQuantity(value, out double actual); + + await That(ok).IsTrue(); + await That(actual).IsEqualTo(expected).Within(0.0001); + } + + [Theory] + [InlineData("206.44 ns", 206.44)] + [InlineData("1,377.70 ns", 1377.70)] + [InlineData("1.00 μs", 1000.0)] + [InlineData("1.00 us", 1000.0)] + [InlineData("2 ms", 2_000_000.0)] + [InlineData("3 s", 3_000_000_000.0)] + [InlineData("500 ps", 0.5)] + public async Task TryParseQuantity_ShouldNormalizeTimeUnitsToNanoseconds(string value, double expected) + { + bool ok = BenchmarkReport.TryParseQuantity(value, out double actual); + + await That(ok).IsTrue(); + await That(actual).IsEqualTo(expected).Within(0.0001); + } + + [Theory] + [InlineData("")] + [InlineData("not a number")] + [InlineData("100 weird")] + public async Task TryParseQuantity_WhenInputIsInvalid_ShouldReturnFalse(string value) + { + bool ok = BenchmarkReport.TryParseQuantity(value, out _); + + await That(ok).IsFalse(); + } } diff --git a/Tests/Build.Tests/BuildBodyTests.cs b/Tests/Build.Tests/BuildBodyTests.cs index 7b3cd611..6549fe22 100644 --- a/Tests/Build.Tests/BuildBodyTests.cs +++ b/Tests/Build.Tests/BuildBodyTests.cs @@ -148,6 +148,31 @@ public async Task BuildBody_WhenTableHasEmptyRowSeparatingParameterGroups_Should await That(body).DoesNotContain($"{Environment.NewLine}|{Environment.NewLine}"); } + [Fact] + public async Task BuildBody_WhenTableHasEmptyRowSeparatingParameterGroups_ShouldEmitEmptyRowMatchingHeaderColumnCount() + { + string[] report = + [ + "| Method | N | Mean | Allocated |", + "|--------|---|-----:|----------:|", + "| Indexer_Mockolate | 1 | 100 ns | 1 KB |", + "| Indexer_Moq | 1 | 200 ns | 2 KB |", + "| | | | |", + "| Indexer_Mockolate | 10 | 1000 ns | 10 KB |", + "| Indexer_Moq | 10 | 2000 ns | 20 KB |", + ]; + BenchmarkReportFile file = new(report, null); + + string body = BenchmarkReport.BuildBody([file,], _columnsToRemove); + + string[] lines = body.Split(Environment.NewLine); + int firstGroupLast = Array.IndexOf(lines, "| Moq | 1 | 200 ns | 2 KB |"); + int secondGroupFirst = Array.IndexOf(lines, "| **Mockolate** | **10** | **1000 ns** | **10 KB** |"); + await That(firstGroupLast).IsGreaterThan(-1); + await That(secondGroupFirst).IsGreaterThan(firstGroupLast); + await That(lines[firstGroupLast + 1]).IsEqualTo("| | | | |"); + } + [Fact] public async Task BuildBody_WhenTableHasEmptyRowSeparatingParameterGroups_ShouldStillStripCommonPrefix() { @@ -195,6 +220,73 @@ await That(body).Contains( "`baseline*` rows show the corresponding Mockolate benchmark"); } + [Fact] + public async Task BuildBody_WithBaseline_ShouldNormalizeUnitDifferencesWhenComputingRatio() + { + string[] report = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 500 ns | 1.00 | 1 KB | 1.00 |", + ]; + string[] baseline = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 1.00 μs | 1.00 | 2048 B | 1.00 |", + ]; + BenchmarkReportFile file = new(report, baseline); + + string body = BenchmarkReport.BuildBody([file,], _columnsToRemove); + + await That(body).Contains("| _baseline*_ | _1.00 μs_ | _2.00_ | _2048 B_ | _2.00_ |"); + } + + [Fact] + public async Task BuildBody_WithBaseline_ShouldRecomputeAllocRatioRelativeToCurrentMockolate() + { + string[] report = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 100 ns | 1.00 | 2 KB | 1.00 |", + ]; + string[] baseline = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 100 ns | 1.00 | 1024 B | 1.00 |", + ]; + BenchmarkReportFile file = new(report, baseline); + + string body = BenchmarkReport.BuildBody([file,], _columnsToRemove); + + await That(body).Contains("| _baseline*_ | _100 ns_ | _1.00_ | _1024 B_ | _0.50_ |"); + } + + [Fact] + public async Task BuildBody_WithBaseline_ShouldRecomputeRatioRelativeToCurrentMockolate() + { + string[] report = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 216.85 ns | 1.00 | 1048 B | 1.00 |", + "| Foo_Moq | 1377.70 ns | 6.36 | 2096 B | 2.00 |", + ]; + string[] baseline = + [ + "| Method | Mean | Ratio | Allocated | Alloc Ratio |", + "|--------|-----:|------:|----------:|------------:|", + "| Foo_Mockolate | 206.44 ns | 1.00 | 1048 B | 1.00 |", + ]; + BenchmarkReportFile file = new(report, baseline); + + string body = BenchmarkReport.BuildBody([file,], _columnsToRemove); + + await That(body).Contains("| _baseline*_ | _206.44 ns_ | _0.95_ | _1048 B_ | _1.00_ |"); + } + [Fact] public async Task BuildBody_WithoutBaseline_ShouldNotEmitBaselineNote() {