Skip to content

feat: implement baseline ratio computation and unit normalization in benchmark reporting #748

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vbreuss merged 3 commits into from
May 1, 2026
Merged

feat: implement baseline ratio computation and unit normalization in benchmark reporting #748

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
991de57
feat: implement baseline ratio computation and unit normalization in …
vbreuss May 1, 2026
36f565e
feat: implement baseline ratio computation and unit normalization in …
vbreuss May 1, 2026
28f6040
Merge remote-tracking branch 'origin/topic/calculate-ratio-for-baseli…
vbreuss May 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 95 additions & 7 deletions Pipeline/BenchmarkReport.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;

Expand All @@ -11,8 +12,6 @@ namespace Build;
/// </summary>
internal static class BenchmarkReport
{
Copy link

Copilot AI May 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DefaultColumnsToRemove was removed from BenchmarkReport, but it is still referenced by the build pipeline (Pipeline/Build.Benchmarks.cs calls BenchmarkReport.DefaultColumnsToRemove). This will break compilation for the Pipeline project. Either restore this constant/field, or update the call site to pass the column list another way (e.g., a local constant or configuration).

Suggested change
{
{
public static readonly string[] DefaultColumnsToRemove = Array.Empty<string>();

Copilot uses AI. Check for mistakes.
public static readonly string[] DefaultColumnsToRemove = ["RatioSD", "Gen0", "Gen1", "Gen2",];

public static string BuildBody(IReadOnlyList<BenchmarkReportFile> files, string[] columnsToRemove)
{
StringBuilder sb = new();
Expand Down Expand Up @@ -121,10 +120,22 @@ internal static void FlushTableBuffer(StringBuilder sb, List<TableRow> tableBuff

string commonPrefix = FindCommonRowPrefix(tableBuffer);

TableRow headerRow = tableBuffer[0].RowIndex == 0 ? tableBuffer[0] : null;
int allocatedIndex = headerRow != null ? FindColumnIndex(headerRow.Tokens, "Allocated") : -1;
int ratioIndex = headerRow != null ? FindColumnIndex(headerRow.Tokens, "Ratio") : -1;
int allocRatioIndex = headerRow != null ? FindColumnIndex(headerRow.Tokens, "Alloc Ratio") : -1;

foreach (TableRow row in tableBuffer)
{
if (row.RowIndex >= BenchmarkTableParser.DataRowStartIndex && row.Tokens.Length == 0)
{
if (headerRow != null && headerRow.Tokens.Length > 0)
{
string[] empty = new string[headerRow.Tokens.Length];
Array.Fill(empty, string.Empty);
sb.AppendLine(JoinTokens(empty));
}

continue;
}

Expand All @@ -140,14 +151,18 @@ internal static void FlushTableBuffer(StringBuilder sb, List<TableRow> tableBuff
string key = string.Join("|", row.Tokens.Take(meanIndex));
if (baselineRows.TryGetValue(key, out string[] baselineTokens) && baselineTokens.Length > 0)
{
string[] modifiedBaseline = new string[baselineTokens.Length];
for (int i = 0; i < baselineTokens.Length; i++)
string[] modifiedBaseline = (string[])baselineTokens.Clone();
RecomputeBaselineRatio(modifiedBaseline, row.Tokens, meanIndex, ratioIndex);
RecomputeBaselineRatio(modifiedBaseline, row.Tokens, allocatedIndex, allocRatioIndex);

string[] italicized = new string[modifiedBaseline.Length];
for (int i = 0; i < modifiedBaseline.Length; i++)
{
string content = i == 0 ? "baseline*" : baselineTokens[i];
modifiedBaseline[i] = string.IsNullOrWhiteSpace(content) ? content : $"_{content}_";
string content = i == 0 ? "baseline*" : modifiedBaseline[i];
italicized[i] = string.IsNullOrWhiteSpace(content) ? content : $"_{content}_";
}

sb.AppendLine(JoinTokens(modifiedBaseline));
sb.AppendLine(JoinTokens(italicized));
anyBaselineInjected = true;
}
}
Expand All @@ -165,6 +180,30 @@ internal static void FlushTableBuffer(StringBuilder sb, List<TableRow> tableBuff
tableBuffer.Clear();
}

internal static void RecomputeBaselineRatio(string[] baselineTokens, string[] mockolateTokens,
int valueIndex, int ratioIndex)
{
if (valueIndex <= 0 || ratioIndex <= 0)
{
return;
}

if (valueIndex >= baselineTokens.Length || valueIndex >= mockolateTokens.Length
|| ratioIndex >= baselineTokens.Length)
{
return;
}

if (!TryParseQuantity(baselineTokens[valueIndex], out double baselineValue)
|| !TryParseQuantity(mockolateTokens[valueIndex], out double mockolateValue)
|| mockolateValue <= 0)
{
return;
}

baselineTokens[ratioIndex] = (baselineValue / mockolateValue).ToString("F2", CultureInfo.InvariantCulture);
}

internal static string[] ApplyHeaderAndPrefixStripping(TableRow row, string commonPrefix)
{
if (commonPrefix == null || row.Tokens.Length == 0)
Expand Down Expand Up @@ -241,6 +280,55 @@ internal static string JoinTokens(string[] tokens)
return sb.ToString();
}

internal static int FindColumnIndex(string[] tokens, string columnName) =>
Array.FindIndex(tokens, t => string.Equals(t, columnName, StringComparison.OrdinalIgnoreCase));

internal static bool TryParseQuantity(string value, out double normalized)
{
normalized = 0;
if (string.IsNullOrWhiteSpace(value))
{
return false;
}

string trimmed = value.Trim();
int spaceIdx = trimmed.LastIndexOf(' ');
string numberPart = spaceIdx > 0 ? trimmed.Substring(0, spaceIdx).Trim() : trimmed;
string unitPart = spaceIdx > 0 ? trimmed.Substring(spaceIdx + 1).Trim() : string.Empty;

if (!double.TryParse(numberPart,
NumberStyles.Float | NumberStyles.AllowThousands,
CultureInfo.InvariantCulture,
out double number))
{
return false;
}

double multiplier = unitPart switch
{
"" => 1,
"ps" => 0.001,
"ns" => 1,
"μs" => 1000,
"us" => 1000,
"ms" => 1_000_000,
"s" => 1_000_000_000,
"B" => 1,
"KB" => 1024,
"MB" => 1024d * 1024,
"GB" => 1024d * 1024 * 1024,
_ => double.NaN,
};

if (double.IsNaN(multiplier))
{
return false;
}

normalized = number * multiplier;
return true;
}

internal static int[] DetermineDroppedColumnIndices(string headerLine, string[] columnsToRemove)
{
string[] tokens = SplitTableRow(headerLine);
Expand Down
89 changes: 89 additions & 0 deletions Tests/Build.Tests/BenchmarkReportHelpersTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,22 @@ public async Task DetermineDroppedColumnIndices_WhenNoColumnsMatch_ShouldReturnE
await That(indices).IsEqualTo(Array.Empty<int>());
}

[Fact]
public async Task FindColumnIndex_ShouldMatchCaseInsensitively()
{
int index = BenchmarkReport.FindColumnIndex(["Method", "Mean", "Alloc Ratio",], "alloc ratio");

await That(index).IsEqualTo(2);
}

[Fact]
public async Task FindColumnIndex_WhenColumnNotFound_ShouldReturnMinusOne()
{
int index = BenchmarkReport.FindColumnIndex(["Method", "Mean",], "Ratio");

await That(index).IsEqualTo(-1);
}

[Fact]
public async Task FindCommonRowPrefix_ShouldIgnoreHeaderAndSeparatorRows()
{
Expand Down Expand Up @@ -141,6 +157,39 @@ public async Task FindCommonRowPrefix_WhenDataRowsDisagree_ShouldReturnNull()
await That(prefix).IsNull();
}

[Fact]
public async Task RecomputeBaselineRatio_ShouldOverwriteRatioWithBaselineDividedByMockolate()
{
string[] baseline = ["Foo_Mockolate", "206.44 ns", "1.00", "1048 B", "1.00",];
string[] mockolate = ["Foo_Mockolate", "216.85 ns", "1.00", "1048 B", "1.00",];

BenchmarkReport.RecomputeBaselineRatio(baseline, mockolate, 1, 2);

await That(baseline[2]).IsEqualTo("0.95");
}

[Fact]
public async Task RecomputeBaselineRatio_WhenRatioIndexIsMissing_ShouldNotMutate()
{
string[] baseline = ["Foo_Mockolate", "206.44 ns",];
string[] mockolate = ["Foo_Mockolate", "216.85 ns",];

BenchmarkReport.RecomputeBaselineRatio(baseline, mockolate, 1, -1);

await That(baseline).IsEqualTo(["Foo_Mockolate", "206.44 ns",]);
}

[Fact]
public async Task RecomputeBaselineRatio_WhenValueCannotBeParsed_ShouldNotMutate()
{
string[] baseline = ["Foo_Mockolate", "junk", "1.00",];
string[] mockolate = ["Foo_Mockolate", "216.85 ns", "1.00",];

BenchmarkReport.RecomputeBaselineRatio(baseline, mockolate, 1, 2);

await That(baseline[2]).IsEqualTo("1.00");
}

[Fact]
public async Task RemoveColumns_ShouldDropTheGivenIndicesAndPreserveTheRest()
{
Expand Down Expand Up @@ -176,4 +225,44 @@ public async Task SplitTableRow_WhenLineHasNoPipe_ShouldReturnEmptyArray()

await That(tokens).IsEqualTo(Array.Empty<string>());
}

[Theory]
[InlineData("1048 B", 1048.0)]
[InlineData("1 KB", 1024.0)]
[InlineData("2 MB", 2.0 * 1024 * 1024)]
[InlineData("3 GB", 3.0 * 1024 * 1024 * 1024)]
public async Task TryParseQuantity_ShouldNormalizeMemoryUnitsToBytes(string value, double expected)
{
bool ok = BenchmarkReport.TryParseQuantity(value, out double actual);

await That(ok).IsTrue();
await That(actual).IsEqualTo(expected).Within(0.0001);
}

[Theory]
[InlineData("206.44 ns", 206.44)]
[InlineData("1,377.70 ns", 1377.70)]
[InlineData("1.00 μs", 1000.0)]
[InlineData("1.00 us", 1000.0)]
[InlineData("2 ms", 2_000_000.0)]
[InlineData("3 s", 3_000_000_000.0)]
[InlineData("500 ps", 0.5)]
public async Task TryParseQuantity_ShouldNormalizeTimeUnitsToNanoseconds(string value, double expected)
{
bool ok = BenchmarkReport.TryParseQuantity(value, out double actual);

await That(ok).IsTrue();
await That(actual).IsEqualTo(expected).Within(0.0001);
}

[Theory]
[InlineData("")]
[InlineData("not a number")]
[InlineData("100 weird")]
public async Task TryParseQuantity_WhenInputIsInvalid_ShouldReturnFalse(string value)
{
bool ok = BenchmarkReport.TryParseQuantity(value, out _);

await That(ok).IsFalse();
}
}
92 changes: 92 additions & 0 deletions Tests/Build.Tests/BuildBodyTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,31 @@ public async Task BuildBody_WhenTableHasEmptyRowSeparatingParameterGroups_Should
await That(body).DoesNotContain($"{Environment.NewLine}|{Environment.NewLine}");
}

[Fact]
public async Task BuildBody_WhenTableHasEmptyRowSeparatingParameterGroups_ShouldEmitEmptyRowMatchingHeaderColumnCount()
{
string[] report =
[
"| Method | N | Mean | Allocated |",
"|--------|---|-----:|----------:|",
"| Indexer_Mockolate | 1 | 100 ns | 1 KB |",
"| Indexer_Moq | 1 | 200 ns | 2 KB |",
"| | | | |",
"| Indexer_Mockolate | 10 | 1000 ns | 10 KB |",
"| Indexer_Moq | 10 | 2000 ns | 20 KB |",
];
BenchmarkReportFile file = new(report, null);

string body = BenchmarkReport.BuildBody([file,], _columnsToRemove);

string[] lines = body.Split(Environment.NewLine);
int firstGroupLast = Array.IndexOf(lines, "| Moq | 1 | 200 ns | 2 KB |");
int secondGroupFirst = Array.IndexOf(lines, "| **Mockolate** | **10** | **1000 ns** | **10 KB** |");
await That(firstGroupLast).IsGreaterThan(-1);
await That(secondGroupFirst).IsGreaterThan(firstGroupLast);
await That(lines[firstGroupLast + 1]).IsEqualTo("| | | | |");
}

[Fact]
public async Task BuildBody_WhenTableHasEmptyRowSeparatingParameterGroups_ShouldStillStripCommonPrefix()
{
Expand Down Expand Up @@ -195,6 +220,73 @@ await That(body).Contains(
"`baseline*` rows show the corresponding Mockolate benchmark");
}

[Fact]
public async Task BuildBody_WithBaseline_ShouldNormalizeUnitDifferencesWhenComputingRatio()
{
string[] report =
[
"| Method | Mean | Ratio | Allocated | Alloc Ratio |",
"|--------|-----:|------:|----------:|------------:|",
"| Foo_Mockolate | 500 ns | 1.00 | 1 KB | 1.00 |",
];
string[] baseline =
[
"| Method | Mean | Ratio | Allocated | Alloc Ratio |",
"|--------|-----:|------:|----------:|------------:|",
"| Foo_Mockolate | 1.00 μs | 1.00 | 2048 B | 1.00 |",
];
BenchmarkReportFile file = new(report, baseline);

string body = BenchmarkReport.BuildBody([file,], _columnsToRemove);

await That(body).Contains("| _baseline*_ | _1.00 μs_ | _2.00_ | _2048 B_ | _2.00_ |");
}

[Fact]
public async Task BuildBody_WithBaseline_ShouldRecomputeAllocRatioRelativeToCurrentMockolate()
{
string[] report =
[
"| Method | Mean | Ratio | Allocated | Alloc Ratio |",
"|--------|-----:|------:|----------:|------------:|",
"| Foo_Mockolate | 100 ns | 1.00 | 2 KB | 1.00 |",
];
string[] baseline =
[
"| Method | Mean | Ratio | Allocated | Alloc Ratio |",
"|--------|-----:|------:|----------:|------------:|",
"| Foo_Mockolate | 100 ns | 1.00 | 1024 B | 1.00 |",
];
BenchmarkReportFile file = new(report, baseline);

string body = BenchmarkReport.BuildBody([file,], _columnsToRemove);

await That(body).Contains("| _baseline*_ | _100 ns_ | _1.00_ | _1024 B_ | _0.50_ |");
}

[Fact]
public async Task BuildBody_WithBaseline_ShouldRecomputeRatioRelativeToCurrentMockolate()
{
string[] report =
[
"| Method | Mean | Ratio | Allocated | Alloc Ratio |",
"|--------|-----:|------:|----------:|------------:|",
"| Foo_Mockolate | 216.85 ns | 1.00 | 1048 B | 1.00 |",
"| Foo_Moq | 1377.70 ns | 6.36 | 2096 B | 2.00 |",
];
string[] baseline =
[
"| Method | Mean | Ratio | Allocated | Alloc Ratio |",
"|--------|-----:|------:|----------:|------------:|",
"| Foo_Mockolate | 206.44 ns | 1.00 | 1048 B | 1.00 |",
];
BenchmarkReportFile file = new(report, baseline);

string body = BenchmarkReport.BuildBody([file,], _columnsToRemove);

await That(body).Contains("| _baseline*_ | _206.44 ns_ | _0.95_ | _1048 B_ | _1.00_ |");
}

[Fact]
public async Task BuildBody_WithoutBaseline_ShouldNotEmitBaselineNote()
{
Expand Down
Loading