perf(pr): median-of-N interleaved A/B gate (validation prototype)

.github/workflows/perf-pr-repeat.yml

@@ -0,0 +1,121 @@
+name: Perf PR (repeat A/B)
+
+# Variance-reduced same-runner A/B (nooga/paserati#21). A single-shot base-vs-head
+# A/B on shared runners is too heavy-tailed to gate on: the register-only anchor
+# is blind to memory-bandwidth contention, so memory-bound families (GetOwn on
+# deep objects, PrototypeMethodAccess chain walks) blow out 14-27% on no-op PRs
+# while the anchor stays flat. This runs N INTERLEAVED base/head snapshots across
+# two pre-built worktrees, then aggregates:
+#   strategy 1 median-of-N   — gate if |median delta| > budget
+#   strategy 2 confirm K-of-N — gate only if a regression REPRODUCES on >=K runs
+# Composes with the min-of-count reducer (#22): min within a capture, median
+# across cycles. Touches perf-pr only — no cmd/bench-ratchet change.
+#
+# Opt-in via the `perf-repeat` label so it never collides with the single-shot
+# `perf` label. workflow_dispatch kept for the eventual upstream form.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+    paths:
+      - 'pkg/**'
+      - 'cmd/**'
+      - 'scripts/ab_repeat.py'
+      - '.github/workflows/perf-pr-repeat.yml'
+  workflow_dispatch:
+    inputs:
+      pr:
+        description: 'PR number to benchmark'
+        required: true
+      n:
+        description: 'repeat count (interleaved cycles; odd)'
+        default: '9'
+      budget:
+        description: 'gate budget %'
+        default: '10'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: perf-pr-repeat-${{ github.event.pull_request.number || github.event.inputs.pr }}
+  cancel-in-progress: true
+
+jobs:
+  bench:
+    if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'perf-repeat')
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Resolve base and head
+        id: refs
+        env:
+          EVENT: ${{ github.event_name }}
+          PR_FROM_EVENT: ${{ github.event.pull_request.number }}
+          PR_FROM_INPUT: ${{ github.event.inputs.pr }}
+          BASE_FROM_EVENT: ${{ github.event.pull_request.base.ref }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          if [ "$EVENT" = "workflow_dispatch" ]; then
+            PR="$PR_FROM_INPUT"
+            [[ "$PR" =~ ^[0-9]+$ ]] || { echo "pr must be numeric" >&2; exit 1; }
+            base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR}" --jq .base.ref)"
+          else
+            PR="$PR_FROM_EVENT"
+            base_ref="$BASE_FROM_EVENT"
+          fi
+          git fetch --no-tags origin "$base_ref" "refs/pull/${PR}/head:pr-head"
+          head_sha="$(git rev-parse pr-head)"
+          base_sha="$(git merge-base "origin/${base_ref}" pr-head)"
+          echo "base=${base_sha}" >> "$GITHUB_OUTPUT"
+          echo "head=${head_sha}" >> "$GITHUB_OUTPUT"
+          echo "Base (merge-base): ${base_sha}"
+          echo "Head:              ${head_sha}"
+
+      - name: Build base and head worktrees
+        env:
+          BASE: ${{ steps.refs.outputs.base }}
+          HEAD: ${{ steps.refs.outputs.head }}
+        run: |
+          set -euo pipefail
+          git worktree add --force ../wt-base "$BASE"
+          git worktree add --force ../wt-head "$HEAD"
+          # Warm each worktree's build cache once so the loop's `go run` is fast.
+          ( cd ../wt-base && go build ./cmd/bench-ratchet )
+          ( cd ../wt-head && go build ./cmd/bench-ratchet )
+
+      - name: Interleaved repeat A/B
+        env:
+          N: ${{ github.event.inputs.n || '9' }}
+          BUDGET: ${{ github.event.inputs.budget || '10' }}
+        run: |
+          set -euo pipefail
+          python3 scripts/ab_repeat.py \
+            --base ../wt-base --head ../wt-head \
+            --n "$N" --count 3 --benchtime 500ms \
+            --budget "$BUDGET" --confirm-k 2 \
+            --out "${RUNNER_TEMP}/ab-out" | tee "${RUNNER_TEMP}/ab-report.txt"
+          {
+            echo '## Repeat A/B (variance-reduced) — same-code probe'
+            echo
+            echo '```'
+            cat "${RUNNER_TEMP}/ab-report.txt"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload aggregate
+        uses: actions/upload-artifact@v4
+        with:
+          name: perf-pr-repeat
+          path: ${{ runner.temp }}/ab-out/aggregate.json
+          if-no-files-found: warn

scripts/ab_repeat.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Interleaved repeated A/B for perf-pr variance reduction (paserati port).
+
+Runs N interleaved base/head benchmark snapshots across two pre-built worktrees
+and aggregates per-family deltas two ways:
+
+  strategy 1 (median-of-N): gate if |median delta| > budget
+  strategy 2 (confirm):     gate if >= K of N runs exceed +budget
+
+Motivation (nooga/paserati#21): a single-shot base-vs-head A/B on shared CI
+runners is too heavy-tailed to gate on — the register-only BenchmarkRatchetAnchor
+is blind to memory-bandwidth contention, so memory-bound families (GetOwn on deep
+objects, PrototypeMethodAccess chain walks) blow out 14-27% on no-op PRs while the
+anchor stays flat. Interleaving + repetition suppresses that.
+
+This composes with the min-of-count reducer (nooga/paserati#22): min collapses
+noise WITHIN a capture (across -count repeats), median collapses it ACROSS the N
+base/head cycles. This driver only reads snapshots and touches no cmd/bench-ratchet
+code.
+
+Each side is snapshotted with `bench-ratchet ... snapshot`, which emits
+ratio_to_anchor per benchmark; delta = head_ratio / base_ratio - 1.
+"""
+import argparse, json, os, statistics, subprocess
+
+
+def _read_snapshot(path):
+    """Return ({family: ratio_to_anchor}, anchor_ns). Handles both the flat
+    baseline (paserati: anchor/benchmarks at top level) and the machines-wrapped
+    snapshot (let-go)."""
+    d = json.load(open(path))
+    node = d
+    if "machines" in d:
+        (_, node), = d["machines"].items()
+    benches = {k: e["ratio_to_anchor"] for k, e in node["benchmarks"].items()}
+    return benches, node["anchor"]["ns_per_op"]
+
+
+def snapshot(worktree, bench_args, out, timeout):
+    subprocess.run(
+        ["go", "run", "./cmd/bench-ratchet", *bench_args,
+         "-timeout", timeout, "-baseline", out, "snapshot"],
+        cwd=worktree, check=True,
+    )
+    return _read_snapshot(out)
+
+
+def _summarize(deltas, budget, confirm_k, strip="github.com/nooga/paserati/"):
+    """Per-family gate rows from {family: [delta% per cycle]}: the median-of-N
+    gate (gate_median = median > budget) and the confirm-K-of-N gate. This is the
+    real gate decision — scripts/test_ab_repeat.py drives it on captured samples
+    to prove single-shot phantoms vanish under median-of-N."""
+    rows = []
+    for fam, ds in deltas.items():
+        med = statistics.median(ds)
+        exceed = sum(1 for d in ds if d > budget)
+        rows.append({
+            "fam": fam.replace(strip, ""),
+            "n": len(ds), "median": med, "worst": max(ds, key=abs),
+            "exceed": exceed, "ds": ds,
+            "gate_median": med > budget,
+            "gate_confirm": exceed >= confirm_k,
+        })
+    rows.sort(key=lambda r: r["median"], reverse=True)
+    return rows
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--base", required=True, help="pre-built base worktree")
+    ap.add_argument("--head", required=True, help="pre-built head worktree")
+    # median-of-N tolerates floor((N-1)/2) contaminated cycles. Observed CI
+    # contamination ran up to 2 of 5 cycles, so N=5 (tolerates 2) sat at the edge
+    # and broke on 1/4 runs; N=7 (tolerates 3) held 0 FP/10. N=9 (tolerates 4)
+    # is the next ODD step for extra margin near benchstat's >=10-sample guidance
+    # — even N is avoided because its median averages the two middle cycles,
+    # reintroducing the mean-like tail sensitivity we want to escape.
+    ap.add_argument("--n", type=int, default=9, help="repeat count (interleaved cycles); odd")
+    ap.add_argument("--profile", default="", help="bench-ratchet -profile (let-go); empty uses --count/--benchtime")
+    ap.add_argument("--count", type=int, default=3, help="go test -count per snapshot")
+    ap.add_argument("--benchtime", default="500ms")
+    ap.add_argument("--budget", type=float, default=10.0, help="gate budget percent")
+    ap.add_argument("--confirm-k", type=int, default=2,
+                    help="runs that must exceed budget to gate (strategy 2)")
+    ap.add_argument("--timeout", default="15m")
+    ap.add_argument("--out", default="ab-out")
+    args = ap.parse_args()
+    os.makedirs(args.out, exist_ok=True)
+
+    bench_args = (["-profile", args.profile] if args.profile
+                  else ["-count", str(args.count), "-benchtime", args.benchtime])
+
+    deltas = {}          # family -> [delta% per cycle]
+    anchors = []         # (base_anchor, head_anchor) per cycle
+    for i in range(1, args.n + 1):
+        print(f"::group::cycle {i}/{args.n}", flush=True)
+        # Counterbalance the measurement order (ABBA): odd cycles bench base
+        # first, even cycles head first. Base and head are identical code, so a
+        # fixed base-then-head order lets any first-vs-second-position drift
+        # (warmup, cache, thermal) masquerade as a consistent head regression —
+        # a systematic bias the median cannot remove. Alternating cancels it.
+        if i % 2 == 1:
+            b, ba = snapshot(args.base, bench_args, f"{args.out}/base_{i}.json", args.timeout)
+            h, ha = snapshot(args.head, bench_args, f"{args.out}/head_{i}.json", args.timeout)
+        else:
+            h, ha = snapshot(args.head, bench_args, f"{args.out}/head_{i}.json", args.timeout)
+            b, ba = snapshot(args.base, bench_args, f"{args.out}/base_{i}.json", args.timeout)
+        anchors.append((ba, ha))
+        for fam in set(b) & set(h):
+            if b[fam]:
+                deltas.setdefault(fam, []).append((h[fam] / b[fam] - 1.0) * 100)
+        print("::endgroup::", flush=True)
+
+    rows = _summarize(deltas, args.budget, args.confirm_k)
+
+    json.dump({"budget": args.budget, "confirm_k": args.confirm_k,
+               "count": args.count, "n": args.n,
+               "anchors": anchors, "rows": rows},
+              open(f"{args.out}/aggregate.json", "w"), indent=2)
+
+    print(f"\n=== interleaved A/B, N={args.n}, count={args.count}, "
+          f"budget={args.budget}%, confirm K={args.confirm_k} ===")
+    print("anchor stability per cycle (base/head ns/op):")
+    for i, (ba, ha) in enumerate(anchors, 1):
+        print(f"  cycle {i}: base {ba:.3f}  head {ha:.3f}  Δ{(ha/ba-1)*100:+.1f}%")
+    print(f"\n{'family':44} {'median%':>8} {'worst%':>7} {'exc':>3}  verdict")
+    print("-" * 82)
+    for r in rows[:14]:
+        v = []
+        if r["gate_median"]: v.append("MEDIAN")
+        if r["gate_confirm"]: v.append("CONFIRM")
+        print(f"{r['fam']:44} {r['median']:+8.2f} {r['worst']:+7.2f} "
+              f"{r['exceed']:3d}  {','.join(v) if v else 'ok'}")
+    med_hits = [r["fam"] for r in rows if r["gate_median"]]
+    conf_hits = [r["fam"] for r in rows if r["gate_confirm"]]
+    print("-" * 82)
+    print(f"strategy 1 (median>{args.budget}%): {len(med_hits)} families gate {med_hits or ''}")
+    print(f"strategy 2 (>={args.confirm_k}/{args.n} exceed {args.budget}%): "
+          f"{len(conf_hits)} families gate {conf_hits or ''}")
+    # Same-code probe → any gate is a FALSE POSITIVE.
+    print(f"\nFALSE-POSITIVE gates at budget {args.budget}%: "
+          f"median={len(med_hits)}  confirm={len(conf_hits)}  "
+          f"(both should be 0 on a no-op PR)")
+
+
+if __name__ == "__main__":
+    main()

scripts/test_ab_repeat.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""Verdict-flip evidence for the median-of-N perf-pr gate (nooga/paserati#21).
+
+Feeds real captured no-op A/B samples through the gate's REAL decision function
+(_summarize) and asserts the flip that motivates the gate:
+
+    single-shot (one A/B cycle)  -> at least one family trips the budget (phantom)
+    median-of-N                  -> zero families gate
+
+The fixtures in testdata/perf_pr_*.json are same-code runs captured on the
+mparrett/paserati fork (base == head), so EVERY gate is by definition a false
+positive. Deterministic, no benchmarking. Parallels the min-of-count reducer's
+real-data verdict test (nooga/paserati#22).
+
+Run: `python3 scripts/test_ab_repeat.py` (prints the flip) or `pytest scripts/`.
+"""
+import glob
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from ab_repeat import _summarize  # the real gate decision  # noqa: E402
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+FIXTURES = sorted(glob.glob(os.path.join(HERE, "testdata", "perf_pr_*.json")))
+
+
+def _load(path):
+    d = json.load(open(path))
+    return d, {r["fam"]: r["ds"] for r in d["rows"]}
+
+
+def test_median_of_n_gates_zero_false_positives():
+    """On same-code runs, median-of-N must gate nothing (0 false positives)."""
+    assert FIXTURES, "no perf_pr_*.json fixtures found"
+    for path in FIXTURES:
+        d, deltas = _load(path)
+        rows = _summarize(deltas, d["budget"], confirm_k=2)
+        fps = [r["fam"] for r in rows if r["gate_median"]]
+        assert fps == [], f"{os.path.basename(path)}: median-of-N false positives {fps}"
+
+
+def test_single_shot_would_false_positive():
+    """A single A/B cycle DOES trip the budget — the very noise the gate removes.
+    Single-shot = each family's most-positive cycle as if it were the lone run."""
+    for path in FIXTURES:
+        d, deltas = _load(path)
+        single = {f: [max(ds)] for f, ds in deltas.items()}
+        rows = _summarize(single, d["budget"], confirm_k=2)
+        phantoms = [r["fam"] for r in rows if r["gate_median"]]
+        assert phantoms, f"{os.path.basename(path)}: expected a single-shot phantom, got none"
+
+
+def _main():
+    for path in FIXTURES:
+        d, deltas = _load(path)
+        med = _summarize(deltas, d["budget"], 2)
+        single = _summarize({f: [max(ds)] for f, ds in deltas.items()}, d["budget"], 2)
+        mfp = [r["fam"] for r in med if r["gate_median"]]
+        sfp = [(r["fam"].split("/")[-1], round(r["median"], 1))
+               for r in single if r["gate_median"]]
+        print(f"{os.path.basename(path)}  N={d['n']} budget={d['budget']}%:")
+        print(f"  single-shot false positives: {len(sfp):2d}  e.g. {sfp[:3]}")
+        print(f"  median-of-N false positives: {len(mfp):2d}  {mfp}")
+    test_median_of_n_gates_zero_false_positives()
+    test_single_shot_would_false_positive()
+    print("\nPASS: single-shot trips the budget on real no-op data; "
+          "median-of-N gates 0 on every captured run.")
+
+
+if __name__ == "__main__":
+    _main()