Skip to content

feat: collect benchmark data into 'benchmarks' branch #762

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vbreuss merged 3 commits into from
May 3, 2026
Merged

feat: collect benchmark data into 'benchmarks' branch #762

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7a03a4f
feat: collect benchmark data into 'benchmarks' branch
vbreuss May 3, 2026
c5acd38
refactor: drop bootstrap-only benchmark helpers
vbreuss May 3, 2026
528d109
Potential fix for pull request finding
vbreuss May 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,30 @@ jobs:
./Artifacts/*
if-no-files-found: ignore

publish-benchmark-report:
name: "Publish Benchmark Report"
needs: [ benchmarks ]
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
permissions:
Comment thread
vbreuss marked this conversation as resolved.
contents: write
env:
DOTNET_NOLOGO: true
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup .NET SDKs
uses: actions/setup-dotnet@v5
with:
dotnet-version: |
10.0.x
- name: Publish benchmark report
run: ./build.sh PublishBenchmarkReport
env:
GithubToken: ${{ secrets.GITHUB_TOKEN }}
WorkflowRunId: ${{ github.run_id }}

Comment on lines +106 to +130
mutation-tests:
name: "Mutation tests"
runs-on: ubuntu-latest
Expand Down
7 changes: 7 additions & 0 deletions .nuke/build.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
"BenchmarkDotNet",
"BenchmarkResult",
"Benchmarks",
"BenchmarkSeedHistory",
"CalculateNugetVersion",
"Clean",
"CodeAnalysis",
Expand All @@ -42,6 +43,7 @@
"MutationTestExecution",
"MutationTests",
"Pack",
"PublishBenchmarkReport",
"Restore",
"UnitTests",
"UpdateReadme"
Expand Down Expand Up @@ -122,6 +124,11 @@
"type": "string",
"description": "Filter for BenchmarkDotNet - Default is '*'"
},
"BenchmarkSeedRunLimit": {
"type": "integer",
"description": "Maximum number of historic Build runs to seed via 'BenchmarkSeedHistory' - Default is 30",
"format": "int32"
},
"Configuration": {
"type": "string",
"description": "Configuration to build - Default is 'Debug' (local) or 'Release' (server)",
Expand Down
210 changes: 210 additions & 0 deletions Pipeline/Build.Benchmarks.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Nuke.Common;
using Nuke.Common.IO;
using Nuke.Common.Tooling;
using Nuke.Common.Tools.DotNet;
using Nuke.Common.Tools.Git;
using Octokit;
using Serilog;
using static Nuke.Common.Tools.DotNet.DotNetTasks;
Expand All @@ -15,8 +18,16 @@ namespace Build;

partial class Build
{
private const string BenchmarkBranch = "benchmarks";
private const string BenchmarkDataPath = "Docs/pages/static/js/data.js";
private const string BenchmarkLimitedDataPath = "Docs/pages/static/js/limited-data.js";
private const int BenchmarkLimit = 50;

[Parameter("Filter for BenchmarkDotNet - Default is '*'")] readonly string BenchmarkFilter = "*";

[Parameter("Maximum number of historic Build runs to seed via 'BenchmarkSeedHistory' - Default is 30")]
readonly int BenchmarkSeedRunLimit = 30;

Target BenchmarkDotNet => _ => _
.Executes(() =>
{
Expand Down Expand Up @@ -126,6 +137,205 @@ partial class Build
.DependsOn(BenchmarkDotNet)
.DependsOn(BenchmarkResult);

Target PublishBenchmarkReport => _ => _
.Description("Aggregates BenchmarkDotNet JSON results from the matrix Benchmarks-* artifacts of the " +
"current workflow run, then commits an updated data.js / limited-data.js to the " +
$"'{BenchmarkBranch}' branch.")
.Requires(() => GithubToken)
.Executes(async () =>
{
await "Benchmarks-".DownloadArtifactsStartingWith(ArtifactsDirectory, GithubToken);

List<string> benchmarkReports = LoadBenchmarkJsonReports(ArtifactsDirectory / "Benchmarks" / "results");
if (benchmarkReports.Count == 0)
{
Log.Warning("Skip benchmark report: no benchmark JSON reports found.");
return;
}

PageBenchmarkReportGenerator.CommitInfo commitInfo = ReadCurrentCommitInfo();
Log.Information(
"Appending benchmark data for commit {Sha} ({Author}, {Date}): {Message}",
commitInfo.Sha, commitInfo.Author, commitInfo.Date, commitInfo.Message);

await BuildExtensions.EnsureBranchExistsAsync(BenchmarkBranch, "main", GithubToken);

BuildExtensions.GithubFile dataFile =
await BuildExtensions.ReadBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, GithubToken);
BuildExtensions.GithubFile limitedFile =
await BuildExtensions.ReadBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, GithubToken);

(string updated, string limited) = PageBenchmarkReportGenerator.Append(
commitInfo,
dataFile?.Content ?? string.Empty,
Comment thread
vbreuss marked this conversation as resolved.
Outdated
benchmarkReports,
BenchmarkLimit);

if (string.IsNullOrWhiteSpace(updated))
{
Log.Information("No changes to publish (commit already recorded).");
return;
}

string commitMessage =
$"Update benchmark for {commitInfo.Sha.Substring(0, 8)}: {commitInfo.Message} by {commitInfo.Author}";
await BuildExtensions.WriteBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, commitMessage, updated,
dataFile?.Sha, GithubToken);
await BuildExtensions.WriteBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, commitMessage,
limited, limitedFile?.Sha, GithubToken);
});

Target BenchmarkSeedHistory => _ => _
.Description("Seeds the benchmark data files on the 'benchmarks' branch with results from past " +
"successful 'Build' workflow runs on main, by downloading their Benchmarks-* artifacts. " +
"One-shot tool; safe to re-run (commits already recorded are skipped).")
.Requires(() => GithubToken)
.Executes(async () =>
{
List<BuildExtensions.WorkflowRunInfo> runs = await BuildExtensions.ListSuccessfulRunsAsync(
"build.yml", "main", BenchmarkSeedRunLimit, GithubToken);
if (runs.Count == 0)
{
Log.Warning("No successful 'Build' runs found on main - nothing to seed.");
return;
}

runs.Reverse();
Log.Information("Seeding from {Count} historical runs (oldest first).", runs.Count);

await BuildExtensions.EnsureBranchExistsAsync(BenchmarkBranch, "main", GithubToken);

BuildExtensions.GithubFile dataFile =
await BuildExtensions.ReadBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, GithubToken);
BuildExtensions.GithubFile limitedFile =
await BuildExtensions.ReadBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, GithubToken);

string accumulatedContent = dataFile?.Content ?? string.Empty;
string accumulatedLimited = limitedFile?.Content ?? string.Empty;
string lastFullSha = dataFile?.Sha;
string lastLimitedSha = limitedFile?.Sha;
int updatedRuns = 0;

foreach (BuildExtensions.WorkflowRunInfo run in runs)
{
AbsolutePath runDirectory = ArtifactsDirectory / "SeedHistory" / run.Id.ToString();
runDirectory.CreateOrCleanDirectory();
try
{
await BuildExtensions.DownloadArtifactsFromRunStartingWith(run.Id, "Benchmarks-",
runDirectory, GithubToken);
}
catch (Exception ex)
{
Log.Warning(ex, "Skipping run #{RunId} ({Sha}): could not download artifacts.", run.Id,
run.Sha?.Substring(0, 8));
continue;
}

List<string> benchmarkReports = LoadBenchmarkJsonReports(runDirectory / "Benchmarks" / "results");
if (benchmarkReports.Count == 0)
{
Log.Information("Run #{RunId} ({Sha}) had no benchmark JSON reports - skipping.", run.Id,
run.Sha?.Substring(0, 8));
continue;
}

PageBenchmarkReportGenerator.CommitInfo commitInfo = new(
run.Sha,
run.Author ?? "unknown",
run.Date ?? string.Empty,
run.Message ?? string.Empty);

(string updated, string limited) = PageBenchmarkReportGenerator.Append(
commitInfo,
accumulatedContent,
benchmarkReports,
BenchmarkLimit);

if (string.IsNullOrWhiteSpace(updated))
{
Log.Information("Run #{RunId} ({Sha}) already recorded - skipping.", run.Id,
run.Sha?.Substring(0, 8));
continue;
}

accumulatedContent = updated;
accumulatedLimited = limited;
updatedRuns++;
Log.Information("Seeded run #{RunId} ({Sha}): {Message}", run.Id,
run.Sha?.Substring(0, 8), commitInfo.Message);
}

if (updatedRuns == 0)
{
Log.Information("No new benchmark data to seed.");
return;
}

string commitMessage = $"Seed historic benchmark data ({updatedRuns} runs)";
await BuildExtensions.WriteBranchFileAsync(BenchmarkDataPath, BenchmarkBranch, commitMessage,
accumulatedContent, lastFullSha, GithubToken);
await BuildExtensions.WriteBranchFileAsync(BenchmarkLimitedDataPath, BenchmarkBranch, commitMessage,
accumulatedLimited, lastLimitedSha, GithubToken);
Log.Information("Seeded benchmark data from {Count} historical runs.", updatedRuns);
});

private static List<string> LoadBenchmarkJsonReports(AbsolutePath resultsDirectory)
{
List<string> reports = new();
if (!Directory.Exists(resultsDirectory))
{
return reports;
}

foreach (string file in Directory.GetFiles(resultsDirectory, "*-report-full-compressed.json"))
Comment thread
vbreuss marked this conversation as resolved.
{
reports.Add(File.ReadAllText(file));
}

return reports;
}

private static PageBenchmarkReportGenerator.CommitInfo ReadCurrentCommitInfo()
{
Output[] lines = GitTasks.Git("log -1").ToArray();
string commitId = null, author = null, date = null, message = null;
foreach (string line in lines.Select(x => x.Text))
{
if (commitId == null && line.StartsWith("commit "))
{
commitId = line.Substring("commit ".Length).Substring(0, 40);
continue;
}

if (author == null && line.StartsWith("Author: "))
{
author = line.Substring("Author: ".Length);
int index = author.IndexOf(" <", StringComparison.Ordinal);
if (index > 0)
{
author = author.Substring(0, index);
}

continue;
}

if (date == null && line.StartsWith("Date: "))
{
date = line.Substring("Date: ".Length);
continue;
}

if (commitId != null && author != null && date != null && !string.IsNullOrWhiteSpace(line))
{
message = line.Trim();
break;
}
}

Comment on lines +201 to +235
return new PageBenchmarkReportGenerator.CommitInfo(commitId, author, date, message);
}

async Task<string> DownloadBaselineBenchmarks(AbsolutePath baselineDirectory)
{
long[] candidateRunIds = await BuildExtensions.FindRecentSuccessfulRunIds("build.yml", "main", 10, GithubToken);
Expand Down
Loading
Loading