Skip to content

fix(bench-ratchet): report out-of-scope baseline entries as one line, not MISSING rows (stacks on #561) #570

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nnunley merged 2 commits into from
Jul 23, 2026
Merged

fix(bench-ratchet): report out-of-scope baseline entries as one line, not MISSING rows (stacks on #561) #570

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e805dbe
perf(bench-ratchet): robust sampling — warmup-discarding median + anc…
nnunley Jul 18, 2026
07e6466
fix(bench-ratchet): report out-of-scope baseline entries as one line,…
nnunley Jul 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 140 additions & 38 deletions cmd/bench-ratchet/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,10 @@ const (
// gate. These metrics are deterministic (no CPU-dependent noise), so the
// budget is tight — it only absorbs rare boundary effects (e.g. a map
// resize landing one element differently).
allocBudget = 0.02
defaultCount = 1
allocBudget = 0.02
// 4 reps minimum: the first is discarded as warmup by reduceSamples,
// leaving >=3 for a meaningful median (a median of 2 doesn't exist).
defaultCount = 4
defaultBenchtime = "1s"
defaultTimeout = "10m"
defaultTags = "gogen_ir"
Expand Down Expand Up @@ -240,7 +242,7 @@ func main() {
if mode == "snapshot" {
writeSnapshot(*baselinePath, current)
} else {
writeOrCheck(*baselinePath, current, "update", *budget, *force, *format)
writeOrCheck(*baselinePath, current, "update", *budget, *force, *format, nil)
}
return
}
Expand Down Expand Up @@ -331,7 +333,7 @@ func main() {
if mode == "snapshot" {
writeSnapshot(*baselinePath, current)
} else {
writeOrCheck(*baselinePath, current, mode, effBudget, *force, *format)
writeOrCheck(*baselinePath, current, mode, effBudget, *force, *format, jobs)
}
}

Expand All @@ -353,8 +355,11 @@ func writeSnapshot(path string, current MachineBaseline) {
fmt.Printf("\nwrote snapshot → %s (%d benchmarks for %s)\n", path, len(current.Benchmarks), key)
}

// writeOrCheck dispatches the post-aggregate action.
func writeOrCheck(baselinePath string, current MachineBaseline, mode string, budget float64, force bool, format string) {
// writeOrCheck dispatches the post-aggregate action. jobs is the capture
// scope of THIS run (nil when aggregating from a pre-recorded .jsonl, where
// the scope is unknown); check reporting uses it to suppress out-of-scope
// baseline entries instead of listing them as MISSING.
func writeOrCheck(baselinePath string, current MachineBaseline, mode string, budget float64, force bool, format string, jobs []captureJob) {
switch mode {
case "show":
switch format {
Expand Down Expand Up @@ -415,7 +420,7 @@ func writeOrCheck(baselinePath string, current MachineBaseline, mode string, bud
// Timing gate: ns/op and ratio_to_anchor are CPU-dependent, so gate
// them ONLY against this machine's own profile.
if prof, ok := baseline.Machines[key]; ok {
if e := compareAndReport(prof, current, budget, format); e != 0 {
if e := compareAndReport(prof, current, budget, format, jobs); e != 0 {
exit = e
}
} else {
Expand Down Expand Up @@ -678,7 +683,7 @@ var profiles = map[string]profile{
// Sized for fast PR feedback, not absolute precision: ~53 cases ×
// count 3 × 500ms keeps a two-pass A/B well under the old full-fleet
// runtime, and the 12% budget sits above the residual sub-µs jitter.
count: 3,
count: 4,
benchtime: "500ms",
budget: 0.12,
jobs: func(tags string) ([]captureJob, error) {
Expand Down Expand Up @@ -731,12 +736,12 @@ func buildJobs(packages, tags string, full, manual bool, filterRE *regexp.Regexp
}
jobs := []captureJob{
{pkg: anchorPackage, tags: tags, filter: filterRE},
{pkg: suitePackage, tags: "", filter: suiteRE, variant: "bytecode", count: 1},
{pkg: suitePackage, tags: "", filter: suiteRE, variant: "ir_bytecode", count: 1, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "gogen_ir", filter: suiteRE, variant: "aot_native", count: 1, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "", filter: suiteTotalRE, variant: "total_bytecode", count: 1},
{pkg: suitePackage, tags: "", filter: suiteTotalRE, variant: "total_ir_bytecode", count: 1, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "gogen_ir", filter: suiteTotalRE, variant: "total_aot_native", count: 1, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "", filter: suiteRE, variant: "bytecode", count: 4},
{pkg: suitePackage, tags: "", filter: suiteRE, variant: "ir_bytecode", count: 4, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "gogen_ir", filter: suiteRE, variant: "aot_native", count: 4, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "", filter: suiteTotalRE, variant: "total_bytecode", count: 4},
{pkg: suitePackage, tags: "", filter: suiteTotalRE, variant: "total_ir_bytecode", count: 4, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "gogen_ir", filter: suiteTotalRE, variant: "total_aot_native", count: 4, env: []string{"LG_SUITE_IR=1"}},
{pkg: irCompilePackage, tags: "", filter: irCompileRE, variant: "bytecode"},
{pkg: irCompilePackage, tags: "gogen_ir", filter: irCompileRE, variant: "gogen_ir"},
{pkg: initPackage, tags: "", filter: initRE, variant: "bytecode"},
Expand Down Expand Up @@ -765,12 +770,12 @@ func buildJobs(packages, tags string, full, manual bool, filterRE *regexp.Regexp
}
jobs := []captureJob{
{pkg: anchorPackage, tags: "", filter: anchorRE},
{pkg: suitePackage, tags: "", filter: suiteRE, variant: "bytecode", count: 1},
{pkg: suitePackage, tags: "", filter: suiteRE, variant: "ir_bytecode", count: 1, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "gogen_ir", filter: suiteRE, variant: "aot_native", count: 1, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "", filter: suiteTotalRE, variant: "total_bytecode", count: 1},
{pkg: suitePackage, tags: "", filter: suiteTotalRE, variant: "total_ir_bytecode", count: 1, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "gogen_ir", filter: suiteTotalRE, variant: "total_aot_native", count: 1, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "", filter: suiteRE, variant: "bytecode", count: 4},
{pkg: suitePackage, tags: "", filter: suiteRE, variant: "ir_bytecode", count: 4, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "gogen_ir", filter: suiteRE, variant: "aot_native", count: 4, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "", filter: suiteTotalRE, variant: "total_bytecode", count: 4},
{pkg: suitePackage, tags: "", filter: suiteTotalRE, variant: "total_ir_bytecode", count: 4, env: []string{"LG_SUITE_IR=1"}},
{pkg: suitePackage, tags: "gogen_ir", filter: suiteTotalRE, variant: "total_aot_native", count: 4, env: []string{"LG_SUITE_IR=1"}},
{pkg: irCompilePackage, tags: "", filter: irCompileRE, variant: "bytecode"},
{pkg: irCompilePackage, tags: "gogen_ir", filter: irCompileRE, variant: "gogen_ir"},
{pkg: initPackage, tags: "", filter: initRE, variant: "bytecode"},
Expand Down Expand Up @@ -927,9 +932,35 @@ func captureOnePackage(pkg string, count int, benchtime, timeout, tags string, f
return written, nil
}

// reduceSamples collapses chronological per-rep measurements into one
// robust value, hyperfine-style: with more than one rep, the FIRST is
// discarded as warmup (cold caches, first-touch var resolution, and
// CPU clock ramp make it the systematically slowest — the suite bench
// visibly climbs across in-process reps), and the median of the rest
// is taken. Median, not mean: a single contention spike or GC pause
// skews an average straight into the stored baseline, which is how a
// concurrently-running pass once manufactured six phantom regressions.
// With one rep (legacy captures, -count 1) the value passes through.
func reduceSamples(vals []float64) float64 {
if len(vals) == 0 {
return 0
}
if len(vals) > 1 {
vals = vals[1:]
}
s := append([]float64(nil), vals...)
sort.Float64s(s)
if n := len(s); n%2 == 1 {
return s[n/2]
} else {
return (s[n/2-1] + s[n/2]) / 2
}
}

// aggregateFromFile reads a .jsonl of StreamRecord lines and returns
// a Baseline computed from them. Same-named records (e.g. multiple
// -count repetitions) are averaged.
// a Baseline computed from them. Same-named records (multiple -count
// repetitions, in capture order) reduce via reduceSamples: first rep
// discarded as warmup, median of the remainder.
func aggregateFromFile(path string) (MachineBaseline, error) {
f, err := os.Open(path)
if err != nil {
Expand All @@ -938,12 +969,11 @@ func aggregateFromFile(path string) (MachineBaseline, error) {
defer f.Close()

type accum struct {
pkg string
name string
count int
nsSum, bytesSum, allocSum float64
iters int64
samples []BenchmarkSample
pkg string
name string
ns, bytes, allocs []float64
iters int64
samples []BenchmarkSample
}
byName := map[string]*accum{}
dec := json.NewDecoder(f)
Expand All @@ -958,10 +988,9 @@ func aggregateFromFile(path string) (MachineBaseline, error) {
a = &accum{pkg: rec.Package, name: rec.Name}
byName[key] = a
}
a.count++
a.nsSum += rec.NSPerOp
a.bytesSum += float64(rec.BytesPerOp)
a.allocSum += float64(rec.AllocsPerOp)
a.ns = append(a.ns, rec.NSPerOp)
a.bytes = append(a.bytes, float64(rec.BytesPerOp))
a.allocs = append(a.allocs, float64(rec.AllocsPerOp))
if rec.Iterations > a.iters {
a.iters = rec.Iterations
}
Expand All @@ -974,9 +1003,9 @@ func aggregateFromFile(path string) (MachineBaseline, error) {
Package: a.pkg,
Name: a.name,
Iterations: a.iters,
NSPerOp: a.nsSum / float64(a.count),
BytesPerOp: int64(a.bytesSum / float64(a.count)),
AllocsPerOp: int64(a.allocSum / float64(a.count)),
NSPerOp: reduceSamples(a.ns),
BytesPerOp: int64(reduceSamples(a.bytes)),
AllocsPerOp: int64(reduceSamples(a.allocs)),
Samples: append([]BenchmarkSample(nil), a.samples...),
})
}
Expand Down Expand Up @@ -1383,9 +1412,46 @@ func printBaseline(b MachineBaseline) {
// non-zero exit code when any benchmark exceeded the regression budget.
// format is "text" (default ANSI terminal) or "markdown" (a single
// GitHub/Slack-friendly table).
func compareAndReport(baseline, current MachineBaseline, budget float64, format string) int {
// jobsSelect reports whether fullName (Package + "." + Name, where Name may
// carry "/sub" segments and a " [variant]" suffix) is selected by any of this
// run's capture jobs. Check reporting uses it to distinguish a benchmark that
// is in scope but absent from the run (deleted/renamed — a real MISSING
// signal) from a baseline entry outside the active profile's scope (e.g. the
// pkg/vm micro fleet when running the fast gate), which was never going to be
// measured and must not be reported row-by-row.
func jobsSelect(jobs []captureJob, fullName string) bool {
for _, j := range jobs {
rest, ok := strings.CutPrefix(fullName, j.pkg+".")
if !ok {
continue
}
// Variant label must agree: variant jobs stamp " [v]" onto the name;
// the variant-free anchor job records no label.
if j.variant != "" {
r2, ok := strings.CutSuffix(rest, " ["+j.variant+"]")
if !ok {
continue
}
rest = r2
} else if strings.Contains(rest, " [") {
continue
}
// go test -bench matches per slash-segment and the job filters are
// family-anchored, so match the family segment only.
family := rest

@mparrett mparrett Jul 18, 2026 •
edited
Loading

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Preserve slash-specific -filter semantics

This truncates the benchmark name to its family before applying the original regexp. For a supported manual filter such as -filter "^BenchmarkMapAssoc/HAMT-Assoc/100$", go test does run that exact sub-benchmark (it applies slash-separated regexp segments), but jobsSelect tests the whole regexp only against BenchmarkMapAssoc and returns false. If that targeted sub-benchmark disappears, the report will classify its baseline row as out of scope and suppress the MISSING signal this change is meant to preserve. Suggest reproducing Go test’s slash-segment matching here, or derive scope from an equivalent matcher.

if i := strings.IndexByte(family, '/'); i >= 0 {
family = family[:i]
}
if j.filter.MatchString(family) {
return true
}
}
return false
}

func compareAndReport(baseline, current MachineBaseline, budget float64, format string, jobs []captureJob) int {
if format == "markdown" {
return compareAndReportMarkdown(baseline, current, budget)
return compareAndReportMarkdown(baseline, current, budget, jobs)
}
fmt.Println()
if baseline.Machine.CPUModel != current.Machine.CPUModel ||
Expand All @@ -1401,6 +1467,20 @@ func compareAndReport(baseline, current MachineBaseline, budget float64, format
anchorDrift := (current.Anchor.NSPerOp - baseline.Anchor.NSPerOp) / baseline.Anchor.NSPerOp
fmt.Printf("anchor: baseline %.3f ns/op, current %.3f ns/op (%+.1f%%)\n",
baseline.Anchor.NSPerOp, current.Anchor.NSPerOp, anchorDrift*100)
// The anchor is the divisor for EVERY normalized ratio: on identical
// hardware+toolchain it should be stable to a few percent. A large
// drift means the anchor ran in a different CPU regime on one side
// (P-core vs E-core scheduling, thermal throttle, a concurrent
// load) — and every REGRESSION/ok verdict below is scaled by the
// same bogus factor. Observed in practice as a bimodal ~1.06 vs
// ~1.85 ns anchor on an M3 manufacturing phantom +38% regressions.
if anchorDrift < -0.15 || anchorDrift > 0.15 {
fmt.Printf("WARNING: anchor moved %+.1f%% on the same machine class — normalized\n", anchorDrift*100)
fmt.Printf(" ratios below are scaled by this factor and NOT trustworthy. Likely a\n")
fmt.Printf(" polluted capture (CPU scheduling/thermal/concurrent load) on one side;\n")
fmt.Printf(" re-run on a quiet machine, and if the drift persists recapture the\n")
fmt.Printf(" baseline (bench-ratchet update -force) from a clean checkout.\n")
}
}
fmt.Println()

Expand All @@ -1412,8 +1492,16 @@ func compareAndReport(baseline, current MachineBaseline, budget float64, format
present bool
}
var drifts []drift
outOfScope := 0
for name, base := range baseline.Benchmarks {
cur, ok := current.Benchmarks[name]
// A baseline entry the active profile never selects (e.g. the full
// pkg/vm fleet while running the fast gate) is out of scope, not
// missing — count it once, don't report it row-by-row.
if !ok && len(jobs) > 0 && !jobsSelect(jobs, name) {
outOfScope++
continue
}
d := drift{
name: name,
baseRatio: base.RatioToAnchor,
Expand Down Expand Up @@ -1471,6 +1559,9 @@ func compareAndReport(baseline, current MachineBaseline, budget float64, format
fmt.Println()
fmt.Printf("summary: %d regression(s) > %.1f%% budget, %d missing, %d new\n",
regressions, budget*100, missing, newCount)
if outOfScope > 0 {
fmt.Printf("(%d baseline benchmark(s) outside this profile's scope — not run, not compared)\n", outOfScope)
}
if regressions > 0 {
return 1
}
Expand Down Expand Up @@ -1537,7 +1628,7 @@ func formatWall(ns float64) string {
//
// The output is also Slack-friendly when wrapped in a code block, since
// Slack renders the pipes monospace.
func compareAndReportMarkdown(baseline, current MachineBaseline, budget float64) int {
func compareAndReportMarkdown(baseline, current MachineBaseline, budget float64, jobs []captureJob) int {
type row struct {
name string
baseR, curR float64
Expand All @@ -1551,12 +1642,20 @@ func compareAndReportMarkdown(baseline, current MachineBaseline, budget float64)
regressions := 0
missing := 0
newCount := 0
outOfScope := 0

for name, base := range baseline.Benchmarks {
seen[name] = true
r := row{name: name, baseR: base.RatioToAnchor, baseNs: base.NSPerOp, bestSinceSHA: base.BestSinceSHA}
cur, ok := current.Benchmarks[name]
if !ok {
// Outside the active profile's scope → never run; count once
// instead of emitting a MISSING row per benchmark. (nil jobs =
// scope unknown, e.g. aggregate-from-file → classic MISSING.)
if len(jobs) > 0 && !jobsSelect(jobs, name) {
outOfScope++
continue
}
r.status = "MISSING"
missing++
rows = append(rows, r)
Expand Down Expand Up @@ -1595,6 +1694,9 @@ func compareAndReportMarkdown(baseline, current MachineBaseline, budget float64)
fmt.Println()
fmt.Printf("**bench-ratchet** — %d regression(s) > %.1f%% budget, %d missing, %d new\n\n",
regressions, budget*100, missing, newCount)
if outOfScope > 0 {
fmt.Printf("_%d baseline benchmark(s) outside this profile's scope — not run, not compared._\n\n", outOfScope)
}
fmt.Printf("- baseline: `%s` / `%s` / `%s`\n",
baseline.Machine.CPUModel, baseline.Machine.GoVersion, baseline.Machine.OS+"-"+baseline.Machine.Arch)
fmt.Printf("- current: `%s` / `%s` / `%s`\n",
Expand Down
Loading
Loading