diff --git a/.github/workflows/perf-pr-repeat.yml b/.github/workflows/perf-pr-repeat.yml new file mode 100644 index 00000000..3a07b973 --- /dev/null +++ b/.github/workflows/perf-pr-repeat.yml @@ -0,0 +1,121 @@ +name: Perf PR (repeat A/B) + +# Variance-reduced same-runner A/B (nooga/paserati#21). A single-shot base-vs-head +# A/B on shared runners is too heavy-tailed to gate on: the register-only anchor +# is blind to memory-bandwidth contention, so memory-bound families (GetOwn on +# deep objects, PrototypeMethodAccess chain walks) blow out 14-27% on no-op PRs +# while the anchor stays flat. This runs N INTERLEAVED base/head snapshots across +# two pre-built worktrees, then aggregates: +# strategy 1 median-of-N — gate if |median delta| > budget +# strategy 2 confirm K-of-N — gate only if a regression REPRODUCES on >=K runs +# Composes with the min-of-count reducer (#22): min within a capture, median +# across cycles. Touches perf-pr only — no cmd/bench-ratchet change. +# +# Opt-in via the `perf-repeat` label so it never collides with the single-shot +# `perf` label. workflow_dispatch kept for the eventual upstream form. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled] + paths: + - 'pkg/**' + - 'cmd/**' + - 'scripts/ab_repeat.py' + - '.github/workflows/perf-pr-repeat.yml' + workflow_dispatch: + inputs: + pr: + description: 'PR number to benchmark' + required: true + n: + description: 'repeat count (interleaved cycles; odd)' + default: '9' + budget: + description: 'gate budget %' + default: '10' + +permissions: + contents: read + +concurrency: + group: perf-pr-repeat-${{ github.event.pull_request.number || github.event.inputs.pr }} + cancel-in-progress: true + +jobs: + bench: + if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'perf-repeat') + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Resolve base and head + id: refs + env: + EVENT: ${{ github.event_name }} + PR_FROM_EVENT: ${{ github.event.pull_request.number }} + PR_FROM_INPUT: ${{ github.event.inputs.pr }} + BASE_FROM_EVENT: ${{ github.event.pull_request.base.ref }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if [ "$EVENT" = "workflow_dispatch" ]; then + PR="$PR_FROM_INPUT" + [[ "$PR" =~ ^[0-9]+$ ]] || { echo "pr must be numeric" >&2; exit 1; } + base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR}" --jq .base.ref)" + else + PR="$PR_FROM_EVENT" + base_ref="$BASE_FROM_EVENT" + fi + git fetch --no-tags origin "$base_ref" "refs/pull/${PR}/head:pr-head" + head_sha="$(git rev-parse pr-head)" + base_sha="$(git merge-base "origin/${base_ref}" pr-head)" + echo "base=${base_sha}" >> "$GITHUB_OUTPUT" + echo "head=${head_sha}" >> "$GITHUB_OUTPUT" + echo "Base (merge-base): ${base_sha}" + echo "Head: ${head_sha}" + + - name: Build base and head worktrees + env: + BASE: ${{ steps.refs.outputs.base }} + HEAD: ${{ steps.refs.outputs.head }} + run: | + set -euo pipefail + git worktree add --force ../wt-base "$BASE" + git worktree add --force ../wt-head "$HEAD" + # Warm each worktree's build cache once so the loop's `go run` is fast. + ( cd ../wt-base && go build ./cmd/bench-ratchet ) + ( cd ../wt-head && go build ./cmd/bench-ratchet ) + + - name: Interleaved repeat A/B + env: + N: ${{ github.event.inputs.n || '9' }} + BUDGET: ${{ github.event.inputs.budget || '10' }} + run: | + set -euo pipefail + python3 scripts/ab_repeat.py \ + --base ../wt-base --head ../wt-head \ + --n "$N" --count 3 --benchtime 500ms \ + --budget "$BUDGET" --confirm-k 2 \ + --out "${RUNNER_TEMP}/ab-out" | tee "${RUNNER_TEMP}/ab-report.txt" + { + echo '## Repeat A/B (variance-reduced) — same-code probe' + echo + echo '```' + cat "${RUNNER_TEMP}/ab-report.txt" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload aggregate + uses: actions/upload-artifact@v4 + with: + name: perf-pr-repeat + path: ${{ runner.temp }}/ab-out/aggregate.json + if-no-files-found: warn diff --git a/scripts/ab_repeat.py b/scripts/ab_repeat.py new file mode 100644 index 00000000..3afdc2e8 --- /dev/null +++ b/scripts/ab_repeat.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Interleaved repeated A/B for perf-pr variance reduction (paserati port). + +Runs N interleaved base/head benchmark snapshots across two pre-built worktrees +and aggregates per-family deltas two ways: + + strategy 1 (median-of-N): gate if |median delta| > budget + strategy 2 (confirm): gate if >= K of N runs exceed +budget + +Motivation (nooga/paserati#21): a single-shot base-vs-head A/B on shared CI +runners is too heavy-tailed to gate on — the register-only BenchmarkRatchetAnchor +is blind to memory-bandwidth contention, so memory-bound families (GetOwn on deep +objects, PrototypeMethodAccess chain walks) blow out 14-27% on no-op PRs while the +anchor stays flat. Interleaving + repetition suppresses that. + +This composes with the min-of-count reducer (nooga/paserati#22): min collapses +noise WITHIN a capture (across -count repeats), median collapses it ACROSS the N +base/head cycles. This driver only reads snapshots and touches no cmd/bench-ratchet +code. + +Each side is snapshotted with `bench-ratchet ... snapshot`, which emits +ratio_to_anchor per benchmark; delta = head_ratio / base_ratio - 1. +""" +import argparse, json, os, statistics, subprocess + + +def _read_snapshot(path): + """Return ({family: ratio_to_anchor}, anchor_ns). Handles both the flat + baseline (paserati: anchor/benchmarks at top level) and the machines-wrapped + snapshot (let-go).""" + d = json.load(open(path)) + node = d + if "machines" in d: + (_, node), = d["machines"].items() + benches = {k: e["ratio_to_anchor"] for k, e in node["benchmarks"].items()} + return benches, node["anchor"]["ns_per_op"] + + +def snapshot(worktree, bench_args, out, timeout): + subprocess.run( + ["go", "run", "./cmd/bench-ratchet", *bench_args, + "-timeout", timeout, "-baseline", out, "snapshot"], + cwd=worktree, check=True, + ) + return _read_snapshot(out) + + +def _summarize(deltas, budget, confirm_k, strip="github.com/nooga/paserati/"): + """Per-family gate rows from {family: [delta% per cycle]}: the median-of-N + gate (gate_median = median > budget) and the confirm-K-of-N gate. This is the + real gate decision — scripts/test_ab_repeat.py drives it on captured samples + to prove single-shot phantoms vanish under median-of-N.""" + rows = [] + for fam, ds in deltas.items(): + med = statistics.median(ds) + exceed = sum(1 for d in ds if d > budget) + rows.append({ + "fam": fam.replace(strip, ""), + "n": len(ds), "median": med, "worst": max(ds, key=abs), + "exceed": exceed, "ds": ds, + "gate_median": med > budget, + "gate_confirm": exceed >= confirm_k, + }) + rows.sort(key=lambda r: r["median"], reverse=True) + return rows + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base", required=True, help="pre-built base worktree") + ap.add_argument("--head", required=True, help="pre-built head worktree") + # median-of-N tolerates floor((N-1)/2) contaminated cycles. Observed CI + # contamination ran up to 2 of 5 cycles, so N=5 (tolerates 2) sat at the edge + # and broke on 1/4 runs; N=7 (tolerates 3) held 0 FP/10. N=9 (tolerates 4) + # is the next ODD step for extra margin near benchstat's >=10-sample guidance + # — even N is avoided because its median averages the two middle cycles, + # reintroducing the mean-like tail sensitivity we want to escape. + ap.add_argument("--n", type=int, default=9, help="repeat count (interleaved cycles); odd") + ap.add_argument("--profile", default="", help="bench-ratchet -profile (let-go); empty uses --count/--benchtime") + ap.add_argument("--count", type=int, default=3, help="go test -count per snapshot") + ap.add_argument("--benchtime", default="500ms") + ap.add_argument("--budget", type=float, default=10.0, help="gate budget percent") + ap.add_argument("--confirm-k", type=int, default=2, + help="runs that must exceed budget to gate (strategy 2)") + ap.add_argument("--timeout", default="15m") + ap.add_argument("--out", default="ab-out") + args = ap.parse_args() + os.makedirs(args.out, exist_ok=True) + + bench_args = (["-profile", args.profile] if args.profile + else ["-count", str(args.count), "-benchtime", args.benchtime]) + + deltas = {} # family -> [delta% per cycle] + anchors = [] # (base_anchor, head_anchor) per cycle + for i in range(1, args.n + 1): + print(f"::group::cycle {i}/{args.n}", flush=True) + # Counterbalance the measurement order (ABBA): odd cycles bench base + # first, even cycles head first. Base and head are identical code, so a + # fixed base-then-head order lets any first-vs-second-position drift + # (warmup, cache, thermal) masquerade as a consistent head regression — + # a systematic bias the median cannot remove. Alternating cancels it. + if i % 2 == 1: + b, ba = snapshot(args.base, bench_args, f"{args.out}/base_{i}.json", args.timeout) + h, ha = snapshot(args.head, bench_args, f"{args.out}/head_{i}.json", args.timeout) + else: + h, ha = snapshot(args.head, bench_args, f"{args.out}/head_{i}.json", args.timeout) + b, ba = snapshot(args.base, bench_args, f"{args.out}/base_{i}.json", args.timeout) + anchors.append((ba, ha)) + for fam in set(b) & set(h): + if b[fam]: + deltas.setdefault(fam, []).append((h[fam] / b[fam] - 1.0) * 100) + print("::endgroup::", flush=True) + + rows = _summarize(deltas, args.budget, args.confirm_k) + + json.dump({"budget": args.budget, "confirm_k": args.confirm_k, + "count": args.count, "n": args.n, + "anchors": anchors, "rows": rows}, + open(f"{args.out}/aggregate.json", "w"), indent=2) + + print(f"\n=== interleaved A/B, N={args.n}, count={args.count}, " + f"budget={args.budget}%, confirm K={args.confirm_k} ===") + print("anchor stability per cycle (base/head ns/op):") + for i, (ba, ha) in enumerate(anchors, 1): + print(f" cycle {i}: base {ba:.3f} head {ha:.3f} Δ{(ha/ba-1)*100:+.1f}%") + print(f"\n{'family':44} {'median%':>8} {'worst%':>7} {'exc':>3} verdict") + print("-" * 82) + for r in rows[:14]: + v = [] + if r["gate_median"]: v.append("MEDIAN") + if r["gate_confirm"]: v.append("CONFIRM") + print(f"{r['fam']:44} {r['median']:+8.2f} {r['worst']:+7.2f} " + f"{r['exceed']:3d} {','.join(v) if v else 'ok'}") + med_hits = [r["fam"] for r in rows if r["gate_median"]] + conf_hits = [r["fam"] for r in rows if r["gate_confirm"]] + print("-" * 82) + print(f"strategy 1 (median>{args.budget}%): {len(med_hits)} families gate {med_hits or ''}") + print(f"strategy 2 (>={args.confirm_k}/{args.n} exceed {args.budget}%): " + f"{len(conf_hits)} families gate {conf_hits or ''}") + # Same-code probe → any gate is a FALSE POSITIVE. + print(f"\nFALSE-POSITIVE gates at budget {args.budget}%: " + f"median={len(med_hits)} confirm={len(conf_hits)} " + f"(both should be 0 on a no-op PR)") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_ab_repeat.py b/scripts/test_ab_repeat.py new file mode 100644 index 00000000..df11ad10 --- /dev/null +++ b/scripts/test_ab_repeat.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Verdict-flip evidence for the median-of-N perf-pr gate (nooga/paserati#21). + +Feeds real captured no-op A/B samples through the gate's REAL decision function +(_summarize) and asserts the flip that motivates the gate: + + single-shot (one A/B cycle) -> at least one family trips the budget (phantom) + median-of-N -> zero families gate + +The fixtures in testdata/perf_pr_*.json are same-code runs captured on the +mparrett/paserati fork (base == head), so EVERY gate is by definition a false +positive. Deterministic, no benchmarking. Parallels the min-of-count reducer's +real-data verdict test (nooga/paserati#22). + +Run: `python3 scripts/test_ab_repeat.py` (prints the flip) or `pytest scripts/`. +""" +import glob +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from ab_repeat import _summarize # the real gate decision # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) +FIXTURES = sorted(glob.glob(os.path.join(HERE, "testdata", "perf_pr_*.json"))) + + +def _load(path): + d = json.load(open(path)) + return d, {r["fam"]: r["ds"] for r in d["rows"]} + + +def test_median_of_n_gates_zero_false_positives(): + """On same-code runs, median-of-N must gate nothing (0 false positives).""" + assert FIXTURES, "no perf_pr_*.json fixtures found" + for path in FIXTURES: + d, deltas = _load(path) + rows = _summarize(deltas, d["budget"], confirm_k=2) + fps = [r["fam"] for r in rows if r["gate_median"]] + assert fps == [], f"{os.path.basename(path)}: median-of-N false positives {fps}" + + +def test_single_shot_would_false_positive(): + """A single A/B cycle DOES trip the budget — the very noise the gate removes. + Single-shot = each family's most-positive cycle as if it were the lone run.""" + for path in FIXTURES: + d, deltas = _load(path) + single = {f: [max(ds)] for f, ds in deltas.items()} + rows = _summarize(single, d["budget"], confirm_k=2) + phantoms = [r["fam"] for r in rows if r["gate_median"]] + assert phantoms, f"{os.path.basename(path)}: expected a single-shot phantom, got none" + + +def _main(): + for path in FIXTURES: + d, deltas = _load(path) + med = _summarize(deltas, d["budget"], 2) + single = _summarize({f: [max(ds)] for f, ds in deltas.items()}, d["budget"], 2) + mfp = [r["fam"] for r in med if r["gate_median"]] + sfp = [(r["fam"].split("/")[-1], round(r["median"], 1)) + for r in single if r["gate_median"]] + print(f"{os.path.basename(path)} N={d['n']} budget={d['budget']}%:") + print(f" single-shot false positives: {len(sfp):2d} e.g. {sfp[:3]}") + print(f" median-of-N false positives: {len(mfp):2d} {mfp}") + test_median_of_n_gates_zero_false_positives() + test_single_shot_would_false_positive() + print("\nPASS: single-shot trips the budget on real no-op data; " + "median-of-N gates 0 on every captured run.") + + +if __name__ == "__main__": + _main() diff --git a/scripts/testdata/perf_pr_n5_run.json b/scripts/testdata/perf_pr_n5_run.json new file mode 100644 index 00000000..e1a48f72 --- /dev/null +++ b/scripts/testdata/perf_pr_n5_run.json @@ -0,0 +1,338 @@ +{ + "n": 5, + "budget": 10.0, + "count": 3, + "source": "captured no-op A/B on mparrett/paserati fork PR #14 (same-code, every gate = false positive)", + "rows": [ + { + "fam": "tests.BenchmarkPrototypeCacheHitRate", + "median": 2.5079855627427428, + "ds": [ + 2.3940072439254445, + -3.7713551902533227, + 6.9826341603268505, + 2.5079855627427428, + 6.143189485420475 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithPrototypeCache/StringPrototypeMethod", + "median": 1.2604456439105016, + "ds": [ + 7.29613521158039, + -3.2073117325255085, + 1.2604456439105016, + -5.250996410060149, + 21.588651718359664 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithDetailedStats/StringPrototypeMethod", + "median": 0.842451566665825, + "ds": [ + 2.7993673947923403, + 0.842451566665825, + -1.8287406320614208, + -2.898260113801965, + 3.7945788087662136 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithPrototypeCache/ObjectPrototypeChain", + "median": 0.7518053817426429, + "ds": [ + -0.9452239547081853, + 3.0260013454539214, + -4.68519671026576, + 0.7518053817426429, + 6.739914726139706 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/Baseline/StringPrototypeMethod", + "median": 0.5851335382550005, + "ds": [ + 0.5851335382550005, + 5.606139017665801, + -2.247801013299622, + -9.318289601799613, + 0.7618621105011991 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithDetailedStats/ObjectPrototypeChain", + "median": 0.5339831606556711, + "ds": [ + 1.4579271742111377, + 0.5339831606556711, + 2.766123780019636, + -3.910778038309437, + -0.026353776496168546 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/Baseline/ObjectPrototypeChain", + "median": 0.42088430551368017, + "ds": [ + 6.11734996750859, + -5.459868393835077, + -1.021114586831362, + 0.42088430551368017, + 5.52892197633883 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=32/first", + "median": 0.227222098462887, + "ds": [ + 0.3588516746411585, + 0.227222098462887, + -0.059772863120155506, + 0.0, + 0.7784431137724424 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=16/last", + "median": 0.11869436201779049, + "ds": [ + -0.1779359430605032, + 0.22562788365600994, + -0.23724792408065243, + 0.11869436201779049, + 0.29726516052317464 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=32/last", + "median": 0.11737089201877549, + "ds": [ + 0.646678424456204, + 0.04580273236987065, + -0.17574692442882123, + 0.11737089201877549, + 0.17615971814444187 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=32/round-robin", + "median": 0.0557724484104849, + "ds": [ + -2.127659574468055, + 2.442510755334526, + 0.6685236768802394, + 0.0557724484104849, + -0.38716814159290847 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=1/first", + "median": 0.040573437922630795, + "ds": [ + 0.013517166801846336, + -0.07107320540159501, + 0.06755843804890915, + 0.040573437922630795, + 0.0811468758452838 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=8/round-robin", + "median": 0.0, + "ds": [ + 0.04161464835621409, + -0.11264097861065636, + 0.0, + -0.04159733777038266, + 0.08322929671242818 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=1/last", + "median": 0.0, + "ds": [ + -0.013203063110633906, + -0.16345335431364738, + 0.0, + 0.01320480654958267, + 0.0528331792365444 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=4/round-robin", + "median": 0.0, + "ds": [ + -0.13504388926401933, + -0.2736318407960403, + 0.0, + 0.13550135501356753, + 0.0 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=8/first", + "median": -0.013509862199412304, + "ds": [ + -0.013509862199412304, + -0.15207476724535818, + -0.013509862199412304, + -0.1620089104900746, + 0.040578926011103356 + ] + }, + { + "fam": "tests.BenchmarkFibPlaceholderRun", + "median": -0.01688787400788705, + "ds": [ + -0.7745466626651099, + 4.87794319086472, + 0.38679816956181945, + -1.7806347150782775, + -0.01688787400788705 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=8/last", + "median": -0.025163563160568447, + "ds": [ + -0.025163563160568447, + 5.156597009205188, + 0.0, + -0.05031446540881834, + -0.07545271629779693 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=4/first", + "median": -0.030534087026490297, + "ds": [ + -0.13489815189532406, + -0.030534087026490297, + -0.10809350087825909, + -0.013511687609768508, + 0.06758583400918727 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=4/last", + "median": -0.046750818139329464, + "ds": [ + -0.18700327255727345, + 0.06921073488095342, + -0.046750818139329464, + 0.0936329588014928, + -0.0467726847521055 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=16/first", + "median": -0.06045949214025237, + "ds": [ + -0.06042296072508169, + 0.11017635952212945, + -0.06045949214025237, + -0.1808318264014619, + -0.18126888217523396 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=1/round-robin", + "median": -0.06302785831339142, + "ds": [ + -0.07563342997606304, + -0.03325476458112542, + -0.05041593143434442, + -0.06302785831339142, + -0.06306760847628601 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=16/round-robin", + "median": -0.11261261261262812, + "ds": [ + 1.573917931422164, + -0.1273394930111693, + -0.6148686416992732, + -0.11261261261262812, + 0.16910935738443378 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=64/last", + "median": -0.1781472684085572, + "ds": [ + -20.87447108603667, + -0.3077314097700845, + -0.11862396204032066, + -0.1781472684085572, + 0.0 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=64/first", + "median": -0.1809408926417233, + "ds": [ + 0.06042296072505948, + -0.2521038698845235, + -0.1809408926417233, + -0.06045949214025237, + -0.18105009052504784 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithDetailedStats/ArrayPrototypeMethod", + "median": -0.19839023279655255, + "ds": [ + -0.19839023279655255, + 0.5100630616673874, + -6.306558855747513, + -1.143083882807594, + -0.04650147234991353 + ] + }, + { + "fam": "tests.BenchmarkMatrixMult", + "median": -0.2547676877116767, + "ds": [ + 0.03402911209080539, + -0.38184622507129173, + -0.9286544507605798, + 0.4418034144256122, + -0.2547676877116767 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/Baseline/ArrayPrototypeMethod", + "median": -1.0038183508879417, + "ds": [ + -22.87370831993094, + 2.3865284695472555, + -1.0038183508879417, + 13.869541382651418, + -1.3023057316398412 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=64/round-robin", + "median": -1.103752759381893, + "ds": [ + 1.3958682300390546, + -1.3816165076258269, + -2.1558872305141197, + 1.2658227848101111, + -1.103752759381893 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithPrototypeCache/ArrayPrototypeMethod", + "median": -1.2763307256412726, + "ds": [ + 10.440230284011221, + -0.9075738451039483, + -4.680182747911122, + -1.2763307256412726, + -1.9503903841270986 + ] + } + ] +} \ No newline at end of file diff --git a/scripts/testdata/perf_pr_n7_run.json b/scripts/testdata/perf_pr_n7_run.json new file mode 100644 index 00000000..e21a3182 --- /dev/null +++ b/scripts/testdata/perf_pr_n7_run.json @@ -0,0 +1,398 @@ +{ + "n": 7, + "budget": 10.0, + "count": 3, + "source": "captured no-op A/B on mparrett/paserati fork PR #14 (same-code, every gate = false positive)", + "rows": [ + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithPrototypeCache/StringPrototypeMethod", + "median": 7.065944797591306, + "ds": [ + -3.4600631565700035, + 7.065944797591306, + -0.9680206243634748, + 7.565587036854993, + 18.880540976021166, + -18.13726646587799, + 8.487707105935316 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/Baseline/ArrayPrototypeMethod", + "median": 1.498550242944785, + "ds": [ + 1.498550242944785, + 0.4142941672890865, + 1.531379220623319, + -18.349385063846846, + 8.49677546121197, + -8.318600224206751, + 4.800211366100471 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/Baseline/StringPrototypeMethod", + "median": 0.7135241977378026, + "ds": [ + 1.8037410665835862, + 0.7135241977378026, + -0.05948553794581368, + 0.906103291677729, + 18.264934950879596, + -2.8117251573270385, + -2.8432166574123063 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=64/round-robin", + "median": 0.4512126339537348, + "ds": [ + 7.3443648116919835, + 0.4512126339537348, + 2.16103732913695, + -4.821498032949556, + -3.2084561450517857, + -0.5745610009789348, + 4.440789473684226 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=32/round-robin", + "median": 0.41675205340954147, + "ds": [ + 0.1427825161571139, + 0.05324813631522041, + 0.41675205340954147, + 1.0340841099224285, + 4.546046291114836, + 9.307533035548122, + -4.032669729453808 + ] + }, + { + "fam": "tests.BenchmarkMatrixMult", + "median": 0.27213957802068034, + "ds": [ + -0.4172721635064103, + 0.27213957802068034, + -0.3912042603154364, + 0.34617042046147795, + 0.4227451302021068, + 0.5629229696328064, + 0.025225289322716726 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=16/first", + "median": 0.22600834492352284, + "ds": [ + 0.2265915968342913, + 0.06954102920722072, + -0.22559513297940592, + 0.9432531342675077, + 0.4530354098868816, + 0.22600834492352284, + -0.9702009702009851 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithDetailedStats/StringPrototypeMethod", + "median": 0.17804981237068418, + "ds": [ + 4.3821633060520115, + -2.9718329009067124, + -1.4448901065903574, + -3.4748937900126675, + 0.38924673259668197, + 8.059598948140367, + 0.17804981237068418 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=8/last", + "median": 0.17376194613378804, + "ds": [ + 0.1777109193662474, + 0.030330603578998527, + -0.14730212077723737, + 0.17376194613378804, + 0.44706045900315683, + 0.17782114108653158, + -0.2119927316777681 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=4/last", + "median": 0.13678935022150007, + "ds": [ + 0.19869689770823928, + 0.05589714924536526, + 0.13678935022150007, + 0.0061538670270255835, + 0.22954277560642833, + 0.25454880509003797, + 0.0 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=4/first", + "median": 0.1018290928650023, + "ds": [ + 0.14686364469780333, + -0.13497300539890844, + 0.10814932701084246, + 0.143679735339175, + -0.09607554757868897, + 0.1018290928650023, + 0.07501875468867159 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=16/round-robin", + "median": 0.08680555555555802, + "ds": [ + 9.791148721871634, + 0.0, + -0.029440549981329944, + 0.11632286244906975, + 0.3308695272981277, + 0.08680555555555802, + -0.2865329512894088 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=1/last", + "median": 0.0553736562266538, + "ds": [ + -0.06949127310869052, + 0.12785907089074566, + 0.0553736562266538, + 0.1595104506229017, + 0.04562097766420692, + 0.07259271750450491, + -0.18450184501843658 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=8/first", + "median": 0.03850508683418408, + "ds": [ + 0.08680555555555802, + 0.030016509080010856, + -0.22144478984964522, + 0.24894489565805422, + 0.03850508683418408, + 0.1919703167217124, + -0.030007501875473075 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=32/first", + "median": 0.021045499342986496, + "ds": [ + 0.021045499342986496, + -0.32829940906107247, + -0.08673026886384383, + 0.17376194613378804, + 0.30541849415204325, + 0.08680555555555802, + -0.6523157208088715 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=1/round-robin", + "median": 0.013273161667126665, + "ds": [ + -0.09908132130538316, + -0.03984063745020583, + -0.03363433702850971, + 0.21370363749988464, + 0.04071961321894246, + 0.04691974037669411, + 0.013273161667126665 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=4/round-robin", + "median": 0.007245296767366405, + "ds": [ + 0.16636581434374964, + -0.07949125596183881, + -0.32480795686495867, + 0.014629679340893986, + 0.2533037213824585, + 0.007245296767366405, + 0.0 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=16/last", + "median": 0.0, + "ds": [ + -0.5272239263803602, + -0.27303754266211344, + 0.3233105097865341, + 0.24260989248852116, + 0.6538955479452158, + -0.3250743026977476, + 0.0 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=1/first", + "median": -0.015001500150013669, + "ds": [ + -0.03329260620300589, + -0.015001500150013669, + -0.13167550940865747, + 0.3542281138794179, + 0.14358305022650342, + 0.19198610485580136, + -0.06000600060006578 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=8/round-robin", + "median": -0.036997431765206645, + "ds": [ + -0.2608905431485953, + -0.2979145978152853, + -0.036997431765206645, + 0.17376194613381024, + 0.17361111111109384, + 0.03698633925115491, + -0.09950248756218638 + ] + }, + { + "fam": "tests.BenchmarkFibPlaceholderRun", + "median": -0.04138611488690991, + "ds": [ + -0.1979025805877166, + -0.0636229429327062, + 0.06664941253762269, + 0.11097172787630782, + 0.3857810066518974, + -0.2818724991755839, + -0.04138611488690991 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=32/last", + "median": -0.10791585603111553, + "ds": [ + -0.10791585603111553, + -2.6649746192893398, + 0.04302722429345707, + -17.352626580557374, + -0.021152876791230568, + 0.34694020287344696, + -0.3878474466709969 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=64/first", + "median": -0.2781641168289384, + "ds": [ + 0.0171557256630539, + -0.2781641168289384, + -0.5724197745013093, + 0.17376194613378804, + -0.3821370781322164, + 0.576740967365974, + -0.48409405255877 + ] + }, + { + "fam": "pkg/vm.BenchmarkGetOwn/n=64/last", + "median": -0.3599039386687264, + "ds": [ + 30.167539465695214, + -0.6844626967830281, + -0.3599039386687264, + -18.615767209134372, + -0.16897989816080372, + 0.430274346068793, + -0.9472259810554884 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithPrototypeCache/ArrayPrototypeMethod", + "median": -0.5910865573793167, + "ds": [ + -1.0706096699136247, + 12.191806100260049, + 2.653880631959482, + -0.5910865573793167, + -4.954457956113001, + -4.024000341412992, + 2.836809945526042 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithDetailedStats/ArrayPrototypeMethod", + "median": -0.7939717364660259, + "ds": [ + -0.7939717364660259, + -1.8566648518480888, + 1.5914210534929696, + -8.024590146208844, + -0.08575763869204112, + 4.539063028131141, + -1.254682006593777 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/Baseline/ObjectPrototypeChain", + "median": -1.1772817852232054, + "ds": [ + -2.040818831233693, + -0.25755834513956355, + -3.2119494407315208, + -3.1196680252969955, + -1.1772817852232054, + 1.795895674329251, + -0.9517209933756932 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithPrototypeCache/ObjectPrototypeChain", + "median": -1.6909420469217618, + "ds": [ + 3.8505686044906007, + 0.16196146907727638, + 0.5075650263826725, + -10.619547943046337, + -1.6909420469217618, + -11.112623890357776, + -3.476112100919182 + ] + }, + { + "fam": "tests.BenchmarkPrototypeCacheHitRate", + "median": -1.8153248656712528, + "ds": [ + -1.8153248656712528, + -6.746020130765807, + -5.394340129221176, + 0.016438017652520287, + 2.3261791424730616, + 1.3062404139275063, + -2.1801287988600926 + ] + }, + { + "fam": "tests.BenchmarkPrototypeMethodAccess/WithDetailedStats/ObjectPrototypeChain", + "median": -2.2147191020335355, + "ds": [ + -2.4112368253708882, + -3.0120015602631556, + -0.4666130532867796, + 2.3218177867841527, + -2.2147191020335355, + -3.6022067786233514, + 0.5053330050154736 + ] + } + ] +} \ No newline at end of file