Skip to content

Commit 9ad22d1

Browse files
aduh95aduh95
authored
tools: add GHA benchmark runner
Signed-off-by: Antoine du Hamel <duhamelantoine1995@gmail.com> PR-URL: #60293 Reviewed-By: Filip Skokan <panva.ip@gmail.com>
1 parent 58544fe commit 9ad22d1

2 files changed

Lines changed: 206 additions & 1 deletion

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
name: Benchmark
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
repo:
7+
type: string
8+
description: GitHub repository to fetch from (default to the current repo)
9+
pr_id:
10+
type: number
11+
required: true
12+
description: The PR to test
13+
commit:
14+
required: true
15+
type: string
16+
description: The expect HEAD of the PR
17+
category:
18+
required: true
19+
type: string
20+
description: The category (or categories) of tests to run, for example buffers, cluster etc. Maps to a folders in node/benchmark
21+
filter:
22+
type: string
23+
description: A substring to restrict the benchmarks to run in a category. e.g. `net-c2c`
24+
runs:
25+
type: number
26+
default: 30
27+
description: How many times to repeat each benchmark
28+
29+
permissions:
30+
contents: read
31+
32+
jobs:
33+
build:
34+
strategy:
35+
fail-fast: true
36+
matrix:
37+
include:
38+
- runner: ubuntu-24.04
39+
system: x86_64-linux
40+
- runner: ubuntu-24.04-arm
41+
system: aarch64-linux
42+
- runner: macos-15-intel
43+
system: x86_64-darwin
44+
- runner: macos-latest
45+
system: aarch64-darwin
46+
name: '${{ matrix.system }}: with shared libraries'
47+
runs-on: ${{ matrix.runner }}
48+
steps:
49+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
50+
with:
51+
repository: ${{ inputs.repo || github.repository }}
52+
ref: refs/pull/${{ inputs.pr_id }}/merge
53+
persist-credentials: false
54+
fetch-depth: 2
55+
56+
- name: Validate PR head and roll back to base commit
57+
run: |
58+
[ "$(git rev-parse HEAD^2)" = "$EXPECTED_SHA" ]
59+
git reset HEAD^ --hard
60+
env:
61+
EXPECTED_SHA: ${{ inputs.commit }}
62+
63+
- uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6
64+
with:
65+
extra_nix_config: sandbox = true
66+
67+
- uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
68+
with:
69+
# We do not pass any `authToken` to avoid polluting the cache with potentially untrusted code.
70+
name: nodejs
71+
72+
- name: Configure sccache
73+
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
74+
with:
75+
script: |
76+
core.exportVariable('SCCACHE_GHA_VERSION', 'on');
77+
core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
78+
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
79+
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
80+
81+
- name: Build Node.js on the base commit
82+
run: |
83+
nix-shell \
84+
-I nixpkgs=./tools/nix/pkgs.nix \
85+
--pure --keep TAR_DIR --keep FLAKY_TESTS \
86+
--keep SCCACHE_GHA_ENABLED --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
87+
--arg useSeparateDerivationForV8 true \
88+
--arg loadJSBuiltinsDynamically false \
89+
--arg ccache "${NIX_SCCACHE:-null}" \
90+
--arg devTools '[]' \
91+
--arg benchmarkTools '[]' \
92+
--run '
93+
make build-ci -j4 V=1
94+
'
95+
mv out/Release/node base_node
96+
97+
- name: Checkout the merge commit
98+
run: git reset FETCH_HEAD --hard
99+
100+
- name: Re-build Node.js on the merge commit
101+
# ccache is disabled here to avoid polluting the cache. Local build outputs should make this build relatively quick anyway.
102+
run: |
103+
nix-shell \
104+
-I nixpkgs=./tools/nix/pkgs.nix \
105+
--pure \
106+
--arg useSeparateDerivationForV8 true \
107+
--arg loadJSBuiltinsDynamically false \
108+
--arg ccache 'null' \
109+
--arg devTools '[]' \
110+
--arg benchmarkTools '[]' \
111+
--run '
112+
make -j4 V=1
113+
'
114+
115+
- name: Run benchmark
116+
run: |
117+
nix-shell \
118+
-I nixpkgs=./tools/nix/pkgs.nix \
119+
--pure --keep FILTER --keep LC_ALL --keep LANG \
120+
--arg loadJSBuiltinsDynamically false \
121+
--arg ccache 'null' \
122+
--arg icu 'null' \
123+
--arg sharedLibDeps '{}' \
124+
--arg devTools '[]' \
125+
--run '
126+
set -o pipefail
127+
./base_node benchmark/compare.js \
128+
--filter "$FILTER" \
129+
--runs ${{ inputs.runs }} \
130+
--old ./base_node --new ./node \
131+
-- ${{ inputs.category }} \
132+
| tee /dev/stderr \
133+
> ${{ matrix.system }}.csv
134+
echo "> [!WARNING] "
135+
echo "> Do not take GHA benchmark results as face value, always confirm them"
136+
echo "> using a dedicated machine, e.g. Jenkins CI."
137+
echo
138+
echo "Benchmark results:"
139+
echo
140+
echo '"'"'```'"'"'
141+
Rscript benchmark/compare.R < ${{ matrix.system }}.csv
142+
echo '"'"'```'"'"'
143+
echo
144+
echo "> [!WARNING] "
145+
echo "> Do not take GHA benchmark results as face value, always confirm them"
146+
echo "> using a dedicated machine, e.g. Jenkins CI."
147+
' | tee /dev/stderr >> "$GITHUB_STEP_SUMMARY"
148+
env:
149+
FILTER: ${{ inputs.filter }}
150+
151+
- name: Upload raw benchmark results
152+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
153+
with:
154+
name: csv-${{ matrix.system }}
155+
path: ${{ matrix.system }}.csv
156+
157+
aggregate-results:
158+
needs: build
159+
name: Aggregate benchmark results
160+
runs-on: ubuntu-slim
161+
steps:
162+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
163+
with:
164+
persist-credentials: false
165+
sparse-checkout: |
166+
benchmark/*.R
167+
tools/nix/*.nix
168+
*.nix
169+
sparse-checkout-cone-mode: false
170+
171+
- name: Download benchmark raw results
172+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
173+
with:
174+
pattern: csv-*
175+
merge-multiple: true
176+
path: raw-results
177+
178+
- uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6
179+
with:
180+
extra_nix_config: sandbox = true
181+
182+
- name: Benchmark results
183+
run: |
184+
nix-shell \
185+
-I nixpkgs=./tools/nix/pkgs.nix \
186+
--pure \
187+
-E '(import <nixpkgs> {}).mkShell { buildInputs = import ./tools/nix/benchmarkTools.nix { withHttpBenchmarkDeps = false; }; }' \
188+
--run '
189+
export LC_ALL=C.UTF-8
190+
echo "> [!WARNING] "
191+
echo "> Do not take GHA benchmark results as face value, always confirm them"
192+
echo "> using a dedicated machine, e.g. Jenkins CI."
193+
echo
194+
echo "Benchmark results:"
195+
echo
196+
echo '"'"'```'"'"'
197+
awk "FNR==1 && NR!=1{next;}{print}" raw-results/*.csv | Rscript benchmark/compare.R
198+
echo '"'"'```'"'"'
199+
echo
200+
echo "> [!WARNING] "
201+
echo "> Do not take GHA benchmark results as face value, always confirm them"
202+
echo "> using a dedicated machine, e.g. Jenkins CI."
203+
' | tee /dev/stderr >> "$GITHUB_STEP_SUMMARY"

‎tools/nix/benchmarkTools.nix‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
{
22
pkgs ? import ./pkgs.nix { },
3+
withHttpBenchmarkDeps ? true,
34
}:
45
[
56
pkgs.R
67
pkgs.rPackages.ggplot2
78
pkgs.rPackages.plyr
8-
pkgs.wrk
99
]
10+
++ pkgs.lib.optional withHttpBenchmarkDeps pkgs.wrk
11+
++ pkgs.lib.optional pkgs.stdenv.buildPlatform.isLinux pkgs.glibcLocales

0 commit comments

Comments
 (0)