diff --git a/cmd/bench-ratchet/main.go b/cmd/bench-ratchet/main.go index 34e6e0bf..c5c2dbdc 100644 --- a/cmd/bench-ratchet/main.go +++ b/cmd/bench-ratchet/main.go @@ -150,15 +150,6 @@ type BenchmarkEntry = perfdata.BenchmarkEntry type BenchmarkSample = perfdata.BenchmarkSample type StreamRecord = perfdata.StreamRecord -// timelineFile represents a parsed timeline snapshot filename. -// Timeline format: TIMESTAMP-SHORTSHA-MACHINE.json -type timelineFile struct { - path string - sha string - machine string - timestamp string -} - // Result is the in-memory parse of one benchmark line. type Result struct { Package string @@ -177,23 +168,28 @@ func (r Result) FullName() string { func main() { var ( - baselinePath = flag.String("baseline", defaultBaselinePath, "baseline JSON path") - budget = flag.Float64("budget", defaultBudget, "fractional regression tolerated before flagging (0.05 = 5%)") - packages = flag.String("packages", "", "space-separated go packages to bench (default: discover)") - count = flag.Int("count", defaultCount, "go test -count") - benchtime = flag.String("benchtime", defaultBenchtime, "go test -benchtime") - filter = flag.String("filter", "", "regexp filter on benchmark names (default: all)") - timeout = flag.String("timeout", defaultTimeout, "go test -timeout per package") - outPath = flag.String("out", "", "capture .jsonl output path (default: docs/perf/.runs/-.jsonl)") - inPath = flag.String("in", "", "aggregate .jsonl input path (default: most recent under docs/perf/.runs/)") - force = flag.Bool("force", false, "with update: bypass the ratchet — write current numbers even where they'd loosen the bar. Use sparingly for accepted regressions.") - shaOverride = flag.String("sha", "", "override the SHA recorded for this run (default: git rev-parse HEAD of cwd). Use when aggregating a capture from a worktree that differs from cwd.") - tags = flag.String("tags", defaultTags, "go test -tags. Default 'gogen_ir' so the lowered-to-Go VM is compiled into the test binary alongside the bytecode VM. Has no effect on releases that pre-date the lowered-Go work (the build tag matches no files there).") - format = flag.String("format", "text", "report format: text (default, ANSI terminal), markdown (GitHub/Slack-friendly table), json (the raw baseline)") - full = flag.Bool("full", false, "run the FULL benchmark profile: pkg/vm fleet under -tags plus jank + IR compile under both VM variants. Slow (~25 min) — for mainline profiling and manual deep-dives.") - profile = flag.String("profile", "", "named benchmark profile (e.g. 'pr-fast'). Mutually exclusive with -packages/-filter/-full. Sets the job list plus default count/benchtime/budget; explicit flags still override.") - wasm = flag.Bool("wasm", false, "run benchmarks under GOOS=js/wasm via the go_js_wasm_exec shim (Node), reporting the machine as js/wasm. Forces -tags off (the wasm bundle ships the bytecode VM, not the lowered-Go path). Slower and noisier than native; for the wasm A/B gate.") - perfDataDir = flag.String("perf-data-dir", "", "seed-baseline only: directory containing perf-data timeline snapshots (e.g., /path/to/perf-data/timeline)") + baselinePath = flag.String("baseline", defaultBaselinePath, "baseline JSON path") + budget = flag.Float64("budget", defaultBudget, "fractional regression tolerated before flagging (0.05 = 5%)") + packages = flag.String("packages", "", "space-separated go packages to bench (default: discover)") + count = flag.Int("count", defaultCount, "go test -count") + benchtime = flag.String("benchtime", defaultBenchtime, "go test -benchtime") + filter = flag.String("filter", "", "regexp filter on benchmark names (default: all)") + timeout = flag.String("timeout", defaultTimeout, "go test -timeout per package") + outPath = flag.String("out", "", "capture .jsonl output path (default: docs/perf/.runs/-.jsonl)") + inPath = flag.String("in", "", "aggregate .jsonl input path (default: most recent under docs/perf/.runs/)") + force = flag.Bool("force", false, "with update: bypass the ratchet — write current numbers even where they'd loosen the bar. Use sparingly for accepted regressions.") + shaOverride = flag.String("sha", "", "override the SHA recorded for this run (default: git rev-parse HEAD of cwd). Use when aggregating a capture from a worktree that differs from cwd.") + tags = flag.String("tags", defaultTags, "go test -tags. Default 'gogen_ir' so the lowered-to-Go VM is compiled into the test binary alongside the bytecode VM. Has no effect on releases that pre-date the lowered-Go work (the build tag matches no files there).") + format = flag.String("format", "text", "report format: text (default, ANSI terminal), markdown (GitHub/Slack-friendly table), json (the raw baseline)") + full = flag.Bool("full", false, "run the FULL benchmark profile: pkg/vm fleet under -tags plus jank + IR compile under both VM variants. Slow (~25 min) — for mainline profiling and manual deep-dives.") + profile = flag.String("profile", "", "named benchmark profile (e.g. 'pr-fast'). Mutually exclusive with -packages/-filter/-full. Sets the job list plus default count/benchtime/budget; explicit flags still override.") + wasm = flag.Bool("wasm", false, "run benchmarks under GOOS=js/wasm via the go_js_wasm_exec shim (Node), reporting the machine as js/wasm. Forces -tags off (the wasm bundle ships the bytecode VM, not the lowered-Go path). Slower and noisier than native; for the wasm A/B gate.") + perfDataDir = flag.String("perf-data-dir", "", "seed-baseline only: directory containing perf-data timeline snapshots (e.g., /path/to/perf-data/timeline)") + seedWindow = flag.Int("seed-window", defaultSeedWindow, "seed-baseline only: how many recent snapshots per machine key to reduce. One snapshot is one CI run; seeding from a single run pins whatever that run happened to measure.") + seedCoherenceTol = flag.Float64("seed-coherence-tolerance", defaultSeedCoherenceTolerance, "seed-baseline only: reject a snapshot whose ratio_to_anchor values sit, in median, further than this fraction off the rest of its window. Catches a mixed capture; deliberately NOT a check on the anchor's absolute drift, which moves without the ratios moving.") + seedIterTol = flag.Float64("seed-iteration-tolerance", defaultSeedIterationTolerance, "seed-baseline only: report a benchmark whose b.N spread across the window exceeds this fraction. Reported, not excluded.") + seedMinIters = flag.Int64("seed-min-iterations", defaultSeedMinIterations, "seed-baseline only: report a benchmark whose median b.N falls below this. Reported, not excluded.") + seedArch = flag.String("seed-arch", defaultSeedArch, "seed-baseline only: architecture to seed from (#651: amd64-only initial seed).") ) flag.Parse() @@ -231,15 +227,24 @@ func main() { return } - // seed-baseline: seed the baseline from perf-data timeline snapshots per #651. - // Filters to amd64-only (per #651 decision), preserves existing M3 profile, - // selects newest snapshot per explicit machine key, and excludes six unstable - // b.N=1 BenchmarkClojureTestSuite* variants. + // seed-baseline: derive the baseline from perf-data timeline snapshots per + // #651. Filters to one architecture, preserves the existing M3 profile, and + // reduces a WINDOW of recent snapshots per machine key rather than trusting + // the newest one — see seed.go for why a single snapshot is not a baseline. if mode == "seed-baseline" { if *perfDataDir == "" { die("seed-baseline requires -perf-data-dir ") } - seedBaseline(*baselinePath, *perfDataDir, "") + if *seedWindow < 1 { + die("-seed-window must be at least 1") + } + seedBaseline(*baselinePath, *perfDataDir, seedOptions{ + window: *seedWindow, + coherenceTolerance: *seedCoherenceTol, + iterationTolerance: *seedIterTol, + minIterations: *seedMinIters, + archPrefix: *seedArch, + }) return } @@ -1851,163 +1856,6 @@ func formatWallMD(ns float64) string { } } -// seedBaseline seeds the baseline from perf-data timeline snapshots per #651. -// Implements decision: amd64-only initial seed, preserve existing M3 profile, -// select newest snapshot independently per explicit machine key. -// Excludes the six unstable b.N=1 BenchmarkClojureTestSuite* variants. -func seedBaseline(baselinePath, perfDataDir, _ string) { - // Read timeline snapshots from perfDataDir - baselineFiles, err := filepath.Glob(filepath.Join(perfDataDir, "*.json")) - if err != nil { - die("list timeline files: %v", err) - } - if len(baselineFiles) == 0 { - die("no timeline snapshots found in %s", perfDataDir) - } - - // Parse filenames to extract SHAs and machine info. - // Timeline format: TIMESTAMP-SHORTSHA-MACHINE.json - var files []timelineFile - for _, f := range baselineFiles { - base := filepath.Base(f) - parts := strings.Split(base, "-") - if len(parts) < 3 { - continue - } - ts := parts[0] // 20260720T221533Z - sha := parts[1] // 9c9a3d636c4e (shortened) - machine := strings.TrimSuffix(strings.Join(parts[2:], "-"), ".json") - files = append(files, timelineFile{ - path: f, - sha: sha, - machine: machine, - timestamp: ts, - }) - } - - // Filter to amd64 machines only (per #651: amd64-only initial seed) - var amd64Files []timelineFile - for _, f := range files { - if strings.HasPrefix(f.machine, "amd64-") { - amd64Files = append(amd64Files, f) - } - } - - if len(amd64Files) == 0 { - die("no amd64 machine snapshots found in %s", perfDataDir) - } - - // Find newest snapshot per explicit machine key (#651 decision). - // newestPerKey[machineKey] = (sha, timestamp, path) - type snapshotInfo struct { - sha string - timestamp string - path string - } - newestPerKey := make(map[string]snapshotInfo) - for _, f := range amd64Files { - key := f.machine // Explicit key from filename (e.g. "amd64-amd-epyc-7763") - if info, exists := newestPerKey[key]; !exists || f.timestamp > info.timestamp { - newestPerKey[key] = snapshotInfo{ - sha: f.sha, - timestamp: f.timestamp, - path: f.path, - } - } - } - - fmt.Printf("bench-ratchet: seed-baseline from %s\n", perfDataDir) - fmt.Printf(" amd64 machines: %d profiles\n", len(newestPerKey)) - - // Read existing baseline to preserve M3 profile - var existingBaseline Baseline - existingM3 := make(map[string]MachineBaseline) // M3 entries to preserve - if data, err := os.ReadFile(baselinePath); err == nil { - if err := json.Unmarshal(data, &existingBaseline); err == nil { - // Extract M3 entries (arm64/Apple M3 variant) - for key, mb := range existingBaseline.Machines { - if strings.Contains(key, "apple-m3") || strings.Contains(mb.Machine.CPUModel, "M3") { - existingM3[key] = mb - fmt.Printf(" preserved M3: %s\n", key) - } - } - } - } - - // Read amd64 snapshots and filter out unstable benchmarks - merged := Baseline{ - Version: schemaVersion, - Machines: make(map[string]MachineBaseline), - } - - for machineKey, info := range newestPerKey { - data, err := os.ReadFile(info.path) - if err != nil { - die("read timeline snapshot %s: %v", info.path, err) - } - - var baseline Baseline - if err := json.Unmarshal(data, &baseline); err != nil { - die("parse timeline snapshot %s: %v", info.path, err) - } - - if len(baseline.Machines) == 0 { - continue - } - - for _, mb := range baseline.Machines { - // Filter out the six unstable benchmarks (b.N=1 suite variants) - mb = filterUnstableBenchmarks(mb) - merged.Machines[perfdata.MachineKey(mb.Machine)] = mb - fmt.Printf(" added: %s (SHA: %s)\n", machineKey, info.sha) - } - } - - // Merge with existing M3 profile - for key, mb := range existingM3 { - merged.Machines[key] = mb - } - - // Write the merged baseline - if err := writeBaseline(baselinePath, merged); err != nil { - die("write baseline: %v", err) - } - fmt.Printf(" wrote baseline → %s (%d machine profiles)\n", - baselinePath, len(merged.Machines)) -} - -// filterUnstableBenchmarks removes the six unstable b.N=1 suite benchmarks. -// Per #651: BenchmarkClojureTestSuite and BenchmarkClojureTestSuiteCompileAndRun -// (each under bytecode, ir_bytecode, aot_native variants) are too noisy to ratchet. -func filterUnstableBenchmarks(mb MachineBaseline) MachineBaseline { - if mb.Benchmarks == nil { - return mb - } - - unstableNames := map[string]bool{ - "github.com/nooga/let-go/test.BenchmarkClojureTestSuite": true, - "github.com/nooga/let-go/test.BenchmarkClojureTestSuiteCompileAndRun": true, - } - - filtered := make(map[string]BenchmarkEntry) - for name, entry := range mb.Benchmarks { - // Check if this benchmark is one of the unstable variants - isUnstable := false - for unstable := range unstableNames { - if strings.HasPrefix(name, unstable) { - isUnstable = true - break - } - } - if !isUnstable { - filtered[name] = entry - } - } - - mb.Benchmarks = filtered - return mb -} - func die(format string, args ...any) { fmt.Fprintf(os.Stderr, "bench-ratchet: "+format+"\n", args...) os.Exit(2) diff --git a/cmd/bench-ratchet/main_test.go b/cmd/bench-ratchet/main_test.go index 463ae8bb..4cc7cf9b 100644 --- a/cmd/bench-ratchet/main_test.go +++ b/cmd/bench-ratchet/main_test.go @@ -444,7 +444,7 @@ func TestSeedBaselineAmd64OnlyPreservesM3(t *testing.T) { } // Run seed-baseline. - seedBaseline(baselineFile, timelineDir, "unused-release-sha") + seedBaseline(baselineFile, timelineDir, defaultSeedOptions()) // Verify the output. data, err := os.ReadFile(baselineFile) diff --git a/cmd/bench-ratchet/seed.go b/cmd/bench-ratchet/seed.go new file mode 100644 index 00000000..49886a8f --- /dev/null +++ b/cmd/bench-ratchet/seed.go @@ -0,0 +1,558 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/nooga/let-go/pkg/perfdata" +) + +// timelineName matches a perf-timeline snapshot filename: +// TIMESTAMP-SHORTSHA-MACHINESLUG.json. +// +// Anchoring the shape here rather than splitting on "-" matters because the +// machine slug is itself full of dashes: a positional split yields a +// plausible-looking but wrong SHA for any name that does not have exactly the +// expected leading fields, and nothing downstream can tell that apart from a +// real one. A non-matching name is skipped and reported instead. +var timelineName = regexp.MustCompile(`^(\d{8}T\d{6}Z)-([0-9a-f]{7,40})-(.+)\.json$`) + +// timelineFile is one parsed snapshot filename. +type timelineFile struct { + path string + sha string + machine string // filename slug, e.g. "amd64-amd-epyc-7763-64-core-processor" + timestamp string +} + +// seedCandidate pairs a snapshot file with the machine profile read out of it. +type seedCandidate struct { + file timelineFile + mb MachineBaseline +} + +// seedOptions are the knobs on `seed-baseline`. Defaults are set in main(). +type seedOptions struct { + // window is how many recent snapshots per machine key participate. One + // snapshot is one CI run, and one CI run is one sample. + window int + // coherenceTolerance rejects a candidate whose ratio_to_anchor values sit, + // in median, further than this fraction off the rest of the window. + coherenceTolerance float64 + // iterationTolerance reports a benchmark whose b.N spread across the + // window exceeds this fraction. + iterationTolerance float64 + // minIterations reports a benchmark whose median b.N falls below this. + minIterations int64 + // archPrefix restricts seeding to one architecture (#651: amd64-only). + archPrefix string +} + +// Defaults for `seed-baseline`, shared by the flag definitions in main() and +// by defaultSeedOptions so the documented behaviour has one source. +const ( + defaultSeedWindow = 5 + defaultSeedCoherenceTolerance = 0.05 + defaultSeedIterationTolerance = 0.10 + defaultSeedMinIterations = 20 + defaultSeedArch = "amd64" +) + +// defaultSeedOptions is the configuration `seed-baseline` runs with when no +// flags override it. +func defaultSeedOptions() seedOptions { + return seedOptions{ + window: defaultSeedWindow, + coherenceTolerance: defaultSeedCoherenceTolerance, + iterationTolerance: defaultSeedIterationTolerance, + minIterations: defaultSeedMinIterations, + archPrefix: defaultSeedArch, + } +} + +// unstableBenchmarks are excluded from the seed by decision, not by +// measurement (#651): BenchmarkClojureTestSuite and its CompileAndRun sibling, +// under each of the bytecode / ir_bytecode / aot_native variants. +// +// This list is authoritative but not self-checking, so seedBaseline reports an +// entry that matched nothing. A filter that has silently stopped applying — +// after a rename, say — still reads as if it filtered. +var unstableBenchmarks = []string{ + "github.com/nooga/let-go/test.BenchmarkClojureTestSuite", + "github.com/nooga/let-go/test.BenchmarkClojureTestSuiteCompileAndRun", +} + +// seedBaseline derives docs/perf/baseline.json from the perf-data timeline. +// +// The numbers stored for a machine tier are medians over the last `window` +// snapshots for that tier, not the newest snapshot. One snapshot is one CI run: +// these distributions are a tight core with a one-sided slow tail, so a single +// run has a real chance of being a tail observation, and a baseline seeded from +// one is wrong in a direction nothing downstream can detect. +// +// Median rather than min across the window: `update` already takes a min over +// history when it ratchets, and seeding with a second minimum would stack two +// of them into a floor no clean run can reach. +// +// Identity (captured_at, captured_at_sha, machine) comes from the newest +// surviving snapshot; the numbers come from the window. The SHA therefore names +// the newest contributing run rather than the sole source of the numbers — the +// seed log prints the whole window so the distinction is visible. +func seedBaseline(baselinePath, perfDataDir string, opt seedOptions) { + paths, err := filepath.Glob(filepath.Join(perfDataDir, "*.json")) + if err != nil { + die("list timeline files: %v", err) + } + if len(paths) == 0 { + die("no timeline snapshots found in %s", perfDataDir) + } + + fmt.Printf("bench-ratchet: seed-baseline from %s\n", perfDataDir) + fmt.Printf(" window %d snapshots per machine key, coherence tolerance ±%.0f%%\n", + opt.window, opt.coherenceTolerance*100) + + var skipped []string + byKey := map[string][]timelineFile{} + for _, p := range paths { + m := timelineName.FindStringSubmatch(filepath.Base(p)) + if m == nil { + skipped = append(skipped, filepath.Base(p)) + continue + } + f := timelineFile{path: p, timestamp: m[1], sha: m[2], machine: m[3]} + if !strings.HasPrefix(f.machine, opt.archPrefix+"-") { + continue + } + byKey[f.machine] = append(byKey[f.machine], f) + } + if len(skipped) > 0 { + fmt.Printf(" skipped %d file(s) not matching TIMESTAMP-SHA-MACHINE.json: %s\n", + len(skipped), strings.Join(truncate(skipped, 3), ", ")) + } + if len(byKey) == 0 { + die("no %s machine snapshots found in %s", opt.archPrefix, perfDataDir) + } + + merged := Baseline{Version: schemaVersion, Machines: map[string]MachineBaseline{}} + for _, slug := range sortedKeys(byKey) { + files := byKey[slug] + // Newest first, then keep at most `window`. + sort.Slice(files, func(i, j int) bool { return files[i].timestamp > files[j].timestamp }) + if len(files) > opt.window { + files = files[:opt.window] + } + mb, key, ok := seedOneMachine(slug, files, opt) + if !ok { + continue + } + merged.Machines[key] = mb + } + + // Preserve the local arm64/Apple M3 profile: it gates developer machines + // and has no counterpart in CI (#651). + if data, err := os.ReadFile(baselinePath); err == nil { + var existing Baseline + if err := json.Unmarshal(data, &existing); err == nil { + for key, mb := range existing.Machines { + if strings.Contains(mb.Machine.CPUModel, "M3") { + merged.Machines[key] = mb + fmt.Printf(" preserved: %s (local, not CI-derived)\n", key) + } + } + } + } + + if len(merged.Machines) == 0 { + die("no machine profiles survived seeding — refusing to write an empty baseline") + } + if err := writeBaseline(baselinePath, merged); err != nil { + die("write baseline: %v", err) + } + fmt.Printf(" wrote baseline → %s (%d machine profiles)\n", baselinePath, len(merged.Machines)) +} + +// seedOneMachine reduces one machine key's window into a single profile. +// Returns the profile and the key to store it under, or ok=false if the tier +// could not be seeded. +func seedOneMachine(slug string, files []timelineFile, opt seedOptions) (MachineBaseline, string, bool) { + var cands []seedCandidate + for _, f := range files { + var b Baseline + data, err := os.ReadFile(f.path) + if err != nil { + die("read timeline snapshot %s: %v", f.path, err) + } + if err := json.Unmarshal(data, &b); err != nil { + die("parse timeline snapshot %s: %v", f.path, err) + } + for _, mb := range b.Machines { + // Enforce the architecture filter on the CONTENT, not just the + // filename. The two have disagreed before (an Intel-slugged file + // carrying an EPYC profile), and the filename is what the log + // prints, so a divergence is otherwise invisible. + if mb.Machine.Arch != opt.archPrefix { + fmt.Printf(" %s: skipping %s profile inside %s\n", + slug, mb.Machine.Arch, filepath.Base(f.path)) + continue + } + if got := slugify(perfdata.MachineKey(mb.Machine)); got != slug { + fmt.Printf(" WARNING: %s is named for %q but carries %q — seeding under the profile it carries\n", + filepath.Base(f.path), slug, got) + } + cands = append(cands, seedCandidate{file: f, mb: mb}) + } + } + if len(cands) == 0 { + fmt.Printf(" %s: no usable profile in %d snapshot(s) — skipped\n", slug, len(files)) + return MachineBaseline{}, "", false + } + + cands, dropped := rejectIncoherent(cands, opt.coherenceTolerance) + for _, d := range dropped { + fmt.Printf(" %s: rejected %s — ratios sit %+.1f%% off the window across %d benchmarks (anchor %+.1f%%)\n", + slug, d.sha, d.ratioOffset*100, d.shared, d.anchorDev*100) + } + if len(cands) == 0 { + fmt.Printf(" %s: no snapshot in the window agrees with the others — skipped\n", slug) + return MachineBaseline{}, "", false + } + + // Newest surviving snapshot supplies identity. + sort.Slice(cands, func(i, j int) bool { return cands[i].file.timestamp > cands[j].file.timestamp }) + newest := cands[0] + + anchorNs := medianOf(mapf(cands, func(c seedCandidate) float64 { return c.mb.Anchor.NSPerOp })) + if anchorNs <= 0 { + fmt.Printf(" %s: window anchor median is %.3f ns/op — skipped\n", slug, anchorNs) + return MachineBaseline{}, "", false + } + + out := MachineBaseline{ + CapturedAt: newest.mb.CapturedAt, + CapturedAtSHA: newest.mb.CapturedAtSHA, + Machine: newest.mb.Machine, + Anchor: AnchorRecord{ + Name: newest.mb.Anchor.Name, + Package: newest.mb.Anchor.Package, + NSPerOp: anchorNs, + Iterations: newest.mb.Anchor.Iterations, + }, + Benchmarks: map[string]BenchmarkEntry{}, + } + + rep := reduceBenchmarks(cands, anchorNs, opt) + out.Benchmarks = rep.entries + + fmt.Printf(" %s: %d benchmarks from %d/%d snapshots (anchor %.3f ns/op, newest %s)\n", + perfdata.MachineKey(out.Machine), len(out.Benchmarks), len(cands), len(files), + anchorNs, newest.file.sha) + rep.report(slug, opt) + + return out, perfdata.MachineKey(out.Machine), true +} + +// coherence measures one candidate's agreement with the rest of its window. +type coherence struct { + sha string + // anchorDev is the anchor's deviation from the window median. Reported + // only — see rejectIncoherent for why it is not a rejection criterion. + anchorDev float64 + // ratioOffset is the median, across shared benchmarks, of this snapshot's + // ratio_to_anchor against the window median for that benchmark. + ratioOffset float64 + shared int +} + +// rejectIncoherent drops candidates whose NORMALIZED numbers disagree with the +// rest of the window. +// +// It deliberately does not gate on the anchor's absolute deviation, which was +// the obvious design and is wrong. Measured over the 24 most recent amd64 +// snapshots in perf-data (2026-08-05): anchor deviation ranges -22.4%..+3.1%, +// while ratio_to_anchor for the same snapshots holds to a median 0.03% and a +// worst 1.75%. The -22.4% snapshot (a588a69d2759, EPYC 9V74) is uniformly +// 22.4% fast in raw ns/op across all 162 of its benchmarks and agrees with its +// window on every ratio to within 0.1% — the whole host was fast and the anchor +// divided that back out, which is what the anchor is for. Rejecting on anchor +// deviation would have discarded three good captures, two of which are the very +// snapshots this baseline is seeded from. +// +// What does need rejecting is the MIXED capture: the anchor caught the slow +// tail and the benchmarks did not, or vice versa. Then every ratio from that +// snapshot is uniformly wrong while its raw numbers look ordinary, and the +// stored floor is off by that amount permanently. A mixed capture shows up +// exactly here, as a whole-snapshot offset in normalized space, well clear of +// the 1.75% the corpus actually exhibits. +// +// Below three candidates there is no median worth testing against, so the +// window passes through unfiltered. +func rejectIncoherent(cands []seedCandidate, tolerance float64) ([]seedCandidate, []coherence) { + if len(cands) < 3 || tolerance <= 0 { + return cands, nil + } + anchorMed := medianOf(mapf(cands, func(c seedCandidate) float64 { return c.mb.Anchor.NSPerOp })) + + // Window median ratio per benchmark, over the candidates that carry it. + perBench := map[string][]float64{} + for _, c := range cands { + for name, e := range c.mb.Benchmarks { + if e.RatioToAnchor > 0 { + perBench[name] = append(perBench[name], e.RatioToAnchor) + } + } + } + windowMed := map[string]float64{} + for name, vals := range perBench { + if len(vals) >= 3 { + windowMed[name] = medianOf(vals) + } + } + + var kept []seedCandidate + var dropped []coherence + for _, c := range cands { + var offsets []float64 + for name, e := range c.mb.Benchmarks { + med, ok := windowMed[name] + if !ok || med <= 0 || e.RatioToAnchor <= 0 { + continue + } + offsets = append(offsets, e.RatioToAnchor/med-1) + } + co := coherence{sha: c.file.sha, shared: len(offsets)} + if anchorMed > 0 { + co.anchorDev = (c.mb.Anchor.NSPerOp - anchorMed) / anchorMed + } + // Too few shared benchmarks to judge: keep it rather than guess. + if len(offsets) < 3 { + kept = append(kept, c) + continue + } + co.ratioOffset = medianOf(offsets) + if co.ratioOffset > tolerance || co.ratioOffset < -tolerance { + dropped = append(dropped, co) + continue + } + kept = append(kept, c) + } + return kept, dropped +} + +// seedReport carries what reduceBenchmarks noticed but did not act on. +type seedReport struct { + entries map[string]BenchmarkEntry + // thin lists benchmarks that appeared in too few snapshots to reduce. + thin []string + // moved lists benchmarks whose b.N spread exceeded the tolerance. + moved []string + // lowN lists benchmarks whose median b.N fell below the floor. + lowN []string + // unmatched lists unstableBenchmarks entries that filtered nothing. + unmatched []string +} + +// reduceBenchmarks medians each benchmark across the surviving window. +// +// The reduction happens in RATIO space, not raw ns/op, and ns_per_op is derived +// back from the reduced ratio and the window's anchor. Measured over the recent +// amd64 corpus (2026-08-05): a snapshot's raw ns/op can sit 22% off its window +// because the host was fast that day, while its ratio_to_anchor holds to 1.75% +// worst case. Reducing the quantity that carries host speed, then dividing, +// imports that speed into the stored floor; reducing the normalized quantity +// does not. The two agree whenever the window is drawn from one host and differ +// exactly when it is not, which is the case a shared runner pool guarantees. +// +// Deriving ns_per_op from the same anchor that is stored also keeps +// entry.ratio_to_anchor == entry.ns_per_op / anchor.ns_per_op true by +// construction, which is what `check` relies on. +func reduceBenchmarks(cands []seedCandidate, anchorNs float64, opt seedOptions) seedReport { + rep := seedReport{entries: map[string]BenchmarkEntry{}} + + type acc struct { + ratios, bytes, allocs []float64 + iters []float64 + } + byName := map[string]*acc{} + matched := map[string]bool{} + for _, c := range cands { + for name, e := range c.mb.Benchmarks { + if markUnstable(name, matched) { + continue + } + a := byName[name] + if a == nil { + a = &acc{} + byName[name] = a + } + // Prefer the recorded ratio; fall back to deriving it against the + // snapshot's OWN anchor, which is the normalization it was captured + // under. + r := e.RatioToAnchor + if r <= 0 && c.mb.Anchor.NSPerOp > 0 { + r = e.NSPerOp / c.mb.Anchor.NSPerOp + } + if r <= 0 { + continue + } + a.ratios = append(a.ratios, r) + a.bytes = append(a.bytes, float64(e.BytesPerOp)) + a.allocs = append(a.allocs, float64(e.AllocsPerOp)) + if n := maxIterations(e); n > 0 { + a.iters = append(a.iters, float64(n)) + } + } + } + for _, prefix := range unstableBenchmarks { + if !matched[prefix] { + rep.unmatched = append(rep.unmatched, prefix) + } + } + + // A benchmark must appear in at least half the surviving window to be + // seeded. One that does not is either newly added or intermittent, and + // seeding it from a single observation is the failure this whole path + // exists to avoid; `check` reports it as NEW instead, which is honest. + quorum := (len(cands) + 1) / 2 + + for _, name := range sortedKeys(byName) { + a := byName[name] + if len(a.ratios) < quorum { + rep.thin = append(rep.thin, fmt.Sprintf("%s (%d/%d)", name, len(a.ratios), len(cands))) + continue + } + ratio := medianOf(a.ratios) + rep.entries[name] = BenchmarkEntry{ + NSPerOp: ratio * anchorNs, + BytesPerOp: int64(medianOf(a.bytes)), + AllocsPerOp: int64(medianOf(a.allocs)), + RatioToAnchor: ratio, + } + + // b.N is an OUTPUT of Go's timing loop (N ≈ benchtime / per-op cost), + // so it moves whenever per-op cost moves. Where iterations share state, + // ns/op is genuinely N-dependent, which makes two snapshots taken at + // different N two different protocols rather than two samples. Report + // it: acting on it would mean silently shrinking the gate. + if len(a.iters) > 1 { + lo, hi := minMax(a.iters) + if lo > 0 && opt.iterationTolerance > 0 && (hi/lo-1) > opt.iterationTolerance { + rep.moved = append(rep.moved, fmt.Sprintf("%s (b.N %.0f→%.0f, %+.0f%%)", name, lo, hi, (hi/lo-1)*100)) + } + if med := medianOf(a.iters); opt.minIterations > 0 && med < float64(opt.minIterations) { + rep.lowN = append(rep.lowN, fmt.Sprintf("%s (b.N %.0f)", name, med)) + } + } + } + return rep +} + +// report prints what the reduction noticed. None of it changes the baseline — +// it changes whether someone reading the seed knows what is in it. +func (r seedReport) report(slug string, opt seedOptions) { + if len(r.thin) > 0 { + fmt.Printf(" %d benchmark(s) below quorum, not seeded: %s\n", + len(r.thin), strings.Join(truncate(r.thin, 3), ", ")) + } + if len(r.moved) > 0 { + fmt.Printf(" b.N moved over %.0f%% across the window for %d benchmark(s): %s\n", + opt.iterationTolerance*100, len(r.moved), strings.Join(truncate(r.moved, 3), ", ")) + } + if len(r.lowN) > 0 { + fmt.Printf(" NOTE: %d benchmark(s) under b.N=%d, seeded anyway: %s\n", + len(r.lowN), opt.minIterations, strings.Join(truncate(r.lowN, 3), ", ")) + } + if len(r.unmatched) > 0 { + fmt.Printf(" WARNING: %d unstable-benchmark prefix(es) matched nothing in %s: %s\n", + len(r.unmatched), slug, strings.Join(r.unmatched, ", ")) + fmt.Printf(" the exclusion list may have gone stale against a rename.\n") + } +} + +// markUnstable records EVERY exclusion prefix that matches name and reports +// whether any did. +// +// Every prefix, not just the first: "…BenchmarkClojureTestSuite" is itself a +// prefix of "…BenchmarkClojureTestSuiteCompileAndRun", so returning on the +// first hit leaves the longer entry permanently unmarked and the staleness +// check then reports a live exclusion as matching nothing on every run. +func markUnstable(name string, matched map[string]bool) bool { + any := false + for _, prefix := range unstableBenchmarks { + if strings.HasPrefix(name, prefix) { + matched[prefix] = true + any = true + } + } + return any +} + +// maxIterations returns the largest b.N recorded for an entry, preferring the +// raw samples when the snapshot retained them. +func maxIterations(e BenchmarkEntry) int64 { + var n int64 + for _, s := range e.Samples { + if s.Iterations > n { + n = s.Iterations + } + } + return n +} + +// medianOf is a plain median. Unlike reduceSamples it does NOT discard a +// warmup rep: that adjustment exists for chronological repetitions inside one +// `go test` process, and snapshots from separate CI runs have no such ordering +// — the first is not systematically the slowest. +func medianOf(vals []float64) float64 { + if len(vals) == 0 { + return 0 + } + s := append([]float64(nil), vals...) + sort.Float64s(s) + if n := len(s); n%2 == 1 { + return s[n/2] + } + n := len(s) + return (s[n/2-1] + s[n/2]) / 2 +} + +func minMax(vals []float64) (float64, float64) { + lo, hi := vals[0], vals[0] + for _, v := range vals[1:] { + if v < lo { + lo = v + } + if v > hi { + hi = v + } + } + return lo, hi +} + +func mapf[T any](in []T, f func(T) float64) []float64 { + out := make([]float64, 0, len(in)) + for _, v := range in { + out = append(out, f(v)) + } + return out +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func truncate(in []string, n int) []string { + if len(in) <= n { + return in + } + return append(append([]string(nil), in[:n]...), fmt.Sprintf("… +%d more", len(in)-n)) +} diff --git a/cmd/bench-ratchet/seed_test.go b/cmd/bench-ratchet/seed_test.go new file mode 100644 index 00000000..377c0c37 --- /dev/null +++ b/cmd/bench-ratchet/seed_test.go @@ -0,0 +1,401 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "math" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/nooga/let-go/pkg/perfdata" +) + +// seedFixture describes one mock timeline snapshot. +type seedFixture struct { + stamp string // filename timestamp, e.g. "20260801T010134Z" + sha string + anchorNs float64 + // benches maps benchmark name to its ns/op in this snapshot. + benches map[string]float64 + // iters maps benchmark name to its b.N in this snapshot (optional). + iters map[string]int64 + arch string + model string +} + +func (f seedFixture) machine() perfdata.Machine { + arch, model := f.arch, f.model + if arch == "" { + arch = "amd64" + } + if model == "" { + model = "AMD EPYC 7763" + } + return perfdata.Machine{OS: "linux", Arch: arch, NumCPU: 16, CPUModel: model, GoVersion: "go1.26.4"} +} + +// writeSeedFixtures materialises snapshots into a fresh timeline dir and +// returns (timelineDir, baselinePath). +func writeSeedFixtures(t *testing.T, fixtures []seedFixture) (string, string) { + t.Helper() + tmp := t.TempDir() + timeline := filepath.Join(tmp, "timeline") + if err := os.MkdirAll(timeline, 0o755); err != nil { + t.Fatal(err) + } + for _, f := range fixtures { + m := f.machine() + entries := map[string]BenchmarkEntry{} + for name, ns := range f.benches { + e := BenchmarkEntry{NSPerOp: ns, RatioToAnchor: ns / f.anchorNs} + if n, ok := f.iters[name]; ok { + e.Samples = []BenchmarkSample{{Iterations: n, NSPerOp: ns}} + } + entries[name] = e + } + b := Baseline{ + Version: schemaVersion, + Machines: map[string]MachineBaseline{ + perfdata.MachineKey(m): { + CapturedAt: f.stamp, + CapturedAtSHA: f.sha, + Machine: m, + Anchor: AnchorRecord{ + Name: anchorName, Package: anchorPackage, + NSPerOp: f.anchorNs, Iterations: 1000000000, + }, + Benchmarks: entries, + }, + }, + } + data, err := json.Marshal(b) + if err != nil { + t.Fatal(err) + } + name := fmt.Sprintf("%s-%s-%s.json", f.stamp, f.sha, slugify(perfdata.MachineKey(m))) + if err := os.WriteFile(filepath.Join(timeline, name), data, 0o644); err != nil { + t.Fatal(err) + } + } + return timeline, filepath.Join(tmp, "baseline.json") +} + +// runSeed executes seedBaseline with stdout captured, returning the parsed +// baseline and everything printed. +func runSeed(t *testing.T, timeline, baselinePath string, opt seedOptions) (Baseline, string) { + t.Helper() + old := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stdout = w + done := make(chan string) + go func() { + b, _ := io.ReadAll(r) + done <- string(b) + }() + func() { + defer func() { + w.Close() + os.Stdout = old + }() + seedBaseline(baselinePath, timeline, opt) + }() + out := <-done + + data, err := os.ReadFile(baselinePath) + if err != nil { + t.Fatalf("read seeded baseline: %v", err) + } + var got Baseline + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("parse seeded baseline: %v", err) + } + return got, out +} + +func onlyMachine(t *testing.T, b Baseline) MachineBaseline { + t.Helper() + if len(b.Machines) != 1 { + t.Fatalf("machines = %d, want 1", len(b.Machines)) + } + for _, mb := range b.Machines { + return mb + } + return MachineBaseline{} +} + +// The point of the window: the newest snapshot does not decide the baseline. +// Five runs of the same benchmark, the newest of which is a slow observation +// inside the anchor tolerance — the stored value must be the window median. +func TestSeedBaselineReducesWindowNotNewest(t *testing.T) { + ns := []float64{112, 100, 101, 99, 100} // newest first once sorted + var fx []seedFixture + for i, v := range ns { + fx = append(fx, seedFixture{ + stamp: fmt.Sprintf("2026080%dT010000Z", 5-i), + sha: fmt.Sprintf("%012d", i), + anchorNs: 1.5, + benches: map[string]float64{"test.BenchmarkA": v}, + }) + } + timeline, baselinePath := writeSeedFixtures(t, fx) + got, _ := runSeed(t, timeline, baselinePath, defaultSeedOptions()) + + mb := onlyMachine(t, got) + e, ok := mb.Benchmarks["test.BenchmarkA"] + if !ok { + t.Fatal("test.BenchmarkA missing from seeded baseline") + } + // median{112,100,101,99,100} = 100; the newest sample (112) is 12% high. + if e.NSPerOp != 100 { + t.Errorf("ns_per_op = %v, want 100 (window median, not the newest sample)", e.NSPerOp) + } + // Identity still comes from the newest surviving snapshot. + if mb.CapturedAtSHA != "000000000000" { + t.Errorf("captured_at_sha = %q, want the newest snapshot's SHA", mb.CapturedAtSHA) + } +} + +// The negative result, kept as a test: a snapshot can sit far off its window's +// ANCHOR and still be a perfectly good capture. Measured over the recent amd64 +// corpus, one snapshot ran 22.4% fast in raw ns/op across all 162 of its +// benchmarks and agreed with its window on every ratio to within 0.1%. Gating +// on anchor deviation would discard it — and two more like it. +func TestSeedBaselineKeepsUniformlyFastSnapshot(t *testing.T) { + // Same ratio (66.67) everywhere; the fast host moves anchor and ns/op together. + fx := []seedFixture{ + {stamp: "20260805T010000Z", sha: "aaaaaaaaaaaa", anchorNs: 1.1, benches: map[string]float64{"test.BenchmarkA": 73.337}}, + {stamp: "20260804T010000Z", sha: "bbbbbbbbbbbb", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100}}, + {stamp: "20260803T010000Z", sha: "cccccccccccc", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100}}, + {stamp: "20260802T010000Z", sha: "dddddddddddd", anchorNs: 1.52, benches: map[string]float64{"test.BenchmarkA": 101.333}}, + {stamp: "20260801T010000Z", sha: "eeeeeeeeeeee", anchorNs: 1.48, benches: map[string]float64{"test.BenchmarkA": 98.667}}, + } + timeline, baselinePath := writeSeedFixtures(t, fx) + got, out := runSeed(t, timeline, baselinePath, defaultSeedOptions()) + + if strings.Contains(out, "rejected") { + t.Errorf("a uniformly-fast host is not a bad capture; output was:\n%s", out) + } + mb := onlyMachine(t, got) + if mb.CapturedAtSHA != "aaaaaaaaaaaa" { + t.Errorf("captured_at_sha = %q, want the newest snapshot kept", mb.CapturedAtSHA) + } + if got := mb.Benchmarks["test.BenchmarkA"].RatioToAnchor; math.Abs(got-66.667) > 0.01 { + t.Errorf("ratio_to_anchor = %v, want ~66.667 (unaffected by host speed)", got) + } +} + +// The case that does need rejecting: the anchor moved and the benchmarks did +// not, so every ratio from this snapshot is uniformly wrong while its raw +// numbers look ordinary. +func TestSeedBaselineRejectsMixedCapture(t *testing.T) { + fx := []seedFixture{ + // anchor 22% fast, benchmark unchanged -> ratio 28% high. + {stamp: "20260805T010000Z", sha: "aaaaaaaaaaaa", anchorNs: 1.17, benches: map[string]float64{"test.BenchmarkA": 100, "test.BenchmarkB": 200, "test.BenchmarkC": 300}}, + {stamp: "20260804T010000Z", sha: "bbbbbbbbbbbb", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100, "test.BenchmarkB": 200, "test.BenchmarkC": 300}}, + {stamp: "20260803T010000Z", sha: "cccccccccccc", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100, "test.BenchmarkB": 200, "test.BenchmarkC": 300}}, + {stamp: "20260802T010000Z", sha: "dddddddddddd", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100, "test.BenchmarkB": 200, "test.BenchmarkC": 300}}, + {stamp: "20260801T010000Z", sha: "eeeeeeeeeeee", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100, "test.BenchmarkB": 200, "test.BenchmarkC": 300}}, + } + timeline, baselinePath := writeSeedFixtures(t, fx) + got, out := runSeed(t, timeline, baselinePath, defaultSeedOptions()) + + if !strings.Contains(out, "rejected aaaaaaaaaaaa") { + t.Errorf("want the mixed capture rejected; output was:\n%s", out) + } + mb := onlyMachine(t, got) + // Identity must come from the newest SURVIVING snapshot. + if mb.CapturedAtSHA != "bbbbbbbbbbbb" { + t.Errorf("captured_at_sha = %q, want bbbbbbbbbbbb (newest survivor)", mb.CapturedAtSHA) + } + if got := mb.Benchmarks["test.BenchmarkA"].RatioToAnchor; math.Abs(got-66.667) > 0.01 { + t.Errorf("ratio_to_anchor = %v, want ~66.667 (the mixed snapshot excluded)", got) + } +} + +// ratio_to_anchor is what `check` compares, so it must agree with the ns/op and +// anchor actually stored beside it — not be reduced independently. +func TestSeedBaselineRatioAgreesWithStoredValues(t *testing.T) { + fx := []seedFixture{ + {stamp: "20260805T010000Z", sha: "aaaaaaaaaaaa", anchorNs: 1.6, benches: map[string]float64{"test.BenchmarkA": 90, "test.BenchmarkB": 300}}, + {stamp: "20260804T010000Z", sha: "bbbbbbbbbbbb", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100, "test.BenchmarkB": 310}}, + {stamp: "20260803T010000Z", sha: "cccccccccccc", anchorNs: 1.4, benches: map[string]float64{"test.BenchmarkA": 110, "test.BenchmarkB": 290}}, + } + timeline, baselinePath := writeSeedFixtures(t, fx) + got, _ := runSeed(t, timeline, baselinePath, defaultSeedOptions()) + + mb := onlyMachine(t, got) + for name, e := range mb.Benchmarks { + want := e.NSPerOp / mb.Anchor.NSPerOp + if math.Abs(e.RatioToAnchor-want) > 1e-9 { + t.Errorf("%s: ratio_to_anchor = %v, want %v (ns_per_op / anchor)", name, e.RatioToAnchor, want) + } + } +} + +// b.N is an output of the timing loop, so it moves when per-op cost moves. +// Reducing across snapshots taken at different N mixes two protocols; the seed +// reports that rather than silently dropping the benchmark. +func TestSeedBaselineReportsMovedIterations(t *testing.T) { + fx := []seedFixture{ + {stamp: "20260805T010000Z", sha: "aaaaaaaaaaaa", anchorNs: 1.5, + benches: map[string]float64{"test.BenchmarkA": 100, "test.BenchmarkSteady": 50}, + iters: map[string]int64{"test.BenchmarkA": 400, "test.BenchmarkSteady": 1000}}, + {stamp: "20260804T010000Z", sha: "bbbbbbbbbbbb", anchorNs: 1.5, + benches: map[string]float64{"test.BenchmarkA": 100, "test.BenchmarkSteady": 50}, + iters: map[string]int64{"test.BenchmarkA": 700, "test.BenchmarkSteady": 1010}}, + {stamp: "20260803T010000Z", sha: "cccccccccccc", anchorNs: 1.5, + benches: map[string]float64{"test.BenchmarkA": 100, "test.BenchmarkSteady": 50}, + iters: map[string]int64{"test.BenchmarkA": 900, "test.BenchmarkSteady": 990}}, + } + timeline, baselinePath := writeSeedFixtures(t, fx) + got, out := runSeed(t, timeline, baselinePath, defaultSeedOptions()) + + if !strings.Contains(out, "b.N moved") || !strings.Contains(out, "test.BenchmarkA") { + t.Errorf("want a b.N movement warning naming test.BenchmarkA; output was:\n%s", out) + } + if strings.Contains(out, "test.BenchmarkSteady (b.N") { + t.Errorf("test.BenchmarkSteady moved 2%% and should not be flagged; output was:\n%s", out) + } + // Reported, not excluded — dropping it would shrink the gate silently. + if _, ok := onlyMachine(t, got).Benchmarks["test.BenchmarkA"]; !ok { + t.Error("test.BenchmarkA should still be seeded; the movement is reported, not acted on") + } +} + +// The exclusion list is a decision, not a measurement, so it cannot notice a +// rename on its own. Seeding says when an entry matched nothing. +func TestSeedBaselineWarnsWhenExclusionMatchesNothing(t *testing.T) { + fx := []seedFixture{ + {stamp: "20260805T010000Z", sha: "aaaaaaaaaaaa", anchorNs: 1.5, + benches: map[string]float64{"test.BenchmarkA": 100}}, + } + timeline, baselinePath := writeSeedFixtures(t, fx) + _, out := runSeed(t, timeline, baselinePath, defaultSeedOptions()) + + if !strings.Contains(out, "matched nothing") { + t.Errorf("want a stale-exclusion warning; output was:\n%s", out) + } + for _, prefix := range unstableBenchmarks { + if !strings.Contains(out, prefix) { + t.Errorf("want %q named in the warning; output was:\n%s", prefix, out) + } + } +} + +// A benchmark present in only a minority of the window has too few samples to +// reduce. `check` reporting it as NEW is more honest than seeding it from one +// observation. +func TestSeedBaselineSkipsBenchmarksBelowQuorum(t *testing.T) { + fx := []seedFixture{ + {stamp: "20260805T010000Z", sha: "aaaaaaaaaaaa", anchorNs: 1.5, + benches: map[string]float64{"test.BenchmarkA": 100, "test.BenchmarkNew": 42}}, + {stamp: "20260804T010000Z", sha: "bbbbbbbbbbbb", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100}}, + {stamp: "20260803T010000Z", sha: "cccccccccccc", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100}}, + {stamp: "20260802T010000Z", sha: "dddddddddddd", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100}}, + {stamp: "20260801T010000Z", sha: "eeeeeeeeeeee", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100}}, + } + timeline, baselinePath := writeSeedFixtures(t, fx) + got, out := runSeed(t, timeline, baselinePath, defaultSeedOptions()) + + mb := onlyMachine(t, got) + if _, ok := mb.Benchmarks["test.BenchmarkNew"]; ok { + t.Error("test.BenchmarkNew appeared in 1 of 5 snapshots and should not be seeded") + } + if _, ok := mb.Benchmarks["test.BenchmarkA"]; !ok { + t.Error("test.BenchmarkA appeared in all 5 snapshots and should be seeded") + } + if !strings.Contains(out, "below quorum") { + t.Errorf("want the skip reported; output was:\n%s", out) + } +} + +// A positional split on "-" yields a plausible-looking wrong SHA for a name +// that does not match the expected shape. Skip and say so instead. +func TestSeedBaselineSkipsMalformedFilenames(t *testing.T) { + fx := []seedFixture{ + {stamp: "20260805T010000Z", sha: "aaaaaaaaaaaa", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100}}, + } + timeline, baselinePath := writeSeedFixtures(t, fx) + if err := os.WriteFile(filepath.Join(timeline, "summary.json"), []byte(`{"version":2}`), 0o644); err != nil { + t.Fatal(err) + } + _, out := runSeed(t, timeline, baselinePath, defaultSeedOptions()) + + if !strings.Contains(out, "summary.json") { + t.Errorf("want summary.json reported as skipped; output was:\n%s", out) + } +} + +// The architecture filter must hold against the file's CONTENT. A snapshot +// named for one machine while carrying another is the divergence that put an +// EPYC profile under an Intel key once already. +func TestSeedBaselineWarnsOnSlugContentMismatch(t *testing.T) { + fx := []seedFixture{ + {stamp: "20260805T010000Z", sha: "aaaaaaaaaaaa", anchorNs: 1.5, benches: map[string]float64{"test.BenchmarkA": 100}}, + } + timeline, baselinePath := writeSeedFixtures(t, fx) + // Rename the file so its slug claims a different CPU than it carries. + old := filepath.Join(timeline, "20260805T010000Z-aaaaaaaaaaaa-amd64-amd-epyc-7763.json") + renamed := filepath.Join(timeline, "20260805T010000Z-aaaaaaaaaaaa-amd64-intel-r-xeon-r-platinum-8573c.json") + if err := os.Rename(old, renamed); err != nil { + t.Fatal(err) + } + got, out := runSeed(t, timeline, baselinePath, defaultSeedOptions()) + + if !strings.Contains(out, "is named for") { + t.Errorf("want a slug/content mismatch warning; output was:\n%s", out) + } + // It is stored under the profile it carries, not the one it is named for. + if _, ok := got.Machines["amd64/AMD EPYC 7763"]; !ok { + t.Errorf("want the profile keyed by its content; got keys %v", got.Machines) + } +} + +// Overlapping prefixes: the shorter exclusion must not shadow the longer one +// into looking stale. +func TestSeedBaselineMarksEveryMatchingExclusion(t *testing.T) { + fx := []seedFixture{ + {stamp: "20260805T010000Z", sha: "aaaaaaaaaaaa", anchorNs: 1.5, benches: map[string]float64{ + "test.BenchmarkA": 100, + "github.com/nooga/let-go/test.BenchmarkClojureTestSuite [bytecode]": 5000, + "github.com/nooga/let-go/test.BenchmarkClojureTestSuiteCompileAndRun [total_bytecode]": 9000, + }}, + } + timeline, baselinePath := writeSeedFixtures(t, fx) + got, out := runSeed(t, timeline, baselinePath, defaultSeedOptions()) + + if strings.Contains(out, "matched nothing") { + t.Errorf("both exclusions are present and should be marked matched; output was:\n%s", out) + } + for name := range onlyMachine(t, got).Benchmarks { + if strings.Contains(name, "ClojureTestSuite") { + t.Errorf("%s should have been excluded from the seed", name) + } + } +} + +func TestMedianOf(t *testing.T) { + cases := []struct { + in []float64 + want float64 + }{ + {nil, 0}, + {[]float64{5}, 5}, + {[]float64{4, 2}, 3}, + {[]float64{9, 1, 5}, 5}, + {[]float64{4, 1, 3, 2}, 2.5}, + // Unlike reduceSamples, no rep is discarded: order must not matter. + {[]float64{100, 1, 1, 1, 1}, 1}, + } + for _, c := range cases { + if got := medianOf(c.in); got != c.want { + t.Errorf("medianOf(%v) = %v, want %v", c.in, got, c.want) + } + } +} diff --git a/docs/perf/ratchet.md b/docs/perf/ratchet.md index 875ad172..8f67ffb1 100644 --- a/docs/perf/ratchet.md +++ b/docs/perf/ratchet.md @@ -1,6 +1,6 @@ --- status: active -last-verified: 2026-08-04 +last-verified: 2026-08-05 authoritative-for: - benchmark-ratchet human-verified: @@ -128,8 +128,8 @@ go run ./cmd/bench-ratchet -baseline docs/perf/historical/v1.8.0.json check ### Seeding from CI The active baseline is seeded from CI timeline snapshots via the `seed-baseline` -command. This generates a baseline by selecting the newest amd64 snapshot per -machine key and merging with any existing local M3 profile: +command. It reduces a WINDOW of recent snapshots per machine key — not the +newest one — and merges the result with any existing local M3 profile: ```sh # Fetch the perf-data branch containing timeline snapshots @@ -142,16 +142,49 @@ bench-ratchet -perf-data-dir /timeline \ ``` The command: -- Scans the timeline directory for snapshot files named `TIMESTAMP-SHORTSHA-MACHINE.json` -- Filters to amd64 machines only (per #651 decision: amd64-only initial seed) -- Selects the newest snapshot independently per explicit machine key +- Scans the timeline directory for snapshot files named `TIMESTAMP-SHORTSHA-MACHINE.json`, reporting any name it cannot parse +- Filters to one architecture (`-seed-arch`, default amd64 per #651), on the file's CONTENT as well as its name +- Takes the newest `-seed-window` snapshots (default 5) per machine key +- Rejects a snapshot whose `ratio_to_anchor` values sit more than `-seed-coherence-tolerance` (default 5%) off the rest of its window +- Medians each benchmark across the survivors **in ratio space**, deriving `ns_per_op` back from the window's anchor +- Skips a benchmark present in fewer than half the surviving snapshots, rather than seeding it from one observation +- Reports `b.N` movement across the window and any exclusion-list entry that matched nothing - Preserves any existing arm64/Apple M3 profile for local developer gating -- Excludes the six unstable b.N=1 BenchmarkClojureTestSuite* variants (too noisy to ratchet) -- Merges into a single baseline where `captured_at_sha` points to the incremental SHA - -The `-release-sha` flag is not required for `seed-baseline`; the baseline gates against -the incremental SHA (newest) for real-time drift tracking. Future work (#597, separate) -will backfill per-tier v1.8.0 release-reference snapshots. +- Excludes the six BenchmarkClojureTestSuite* variants (too noisy to ratchet, per #651) + +`captured_at_sha` names the newest *surviving* snapshot in the window, which is +the identity of the profile rather than the sole source of its numbers. The seed +log prints the window size and how many snapshots contributed. + +Future work (#597, separate) will backfill per-tier v1.8.0 release-reference +snapshots. + +### Why a window, and why not gate on the anchor + +**One snapshot is one CI run, and one CI run is one sample.** Seeded from the +newest snapshot versus a median of five, on the same corpus (2026-08-05): 22.6% +of the 758 (tier, benchmark) floors differ by more than the 5% regression +budget, and 11.3% by more than 10%. Part of that is real code movement across +the window and part is sampling — but either way, seeding from one snapshot sets +a fifth of the gate's thresholds from a single observation of it. + +**The reduction happens in ratio space** because raw `ns_per_op` carries host +speed and `ratio_to_anchor` does not. Over the 24 most recent amd64 snapshots, +anchor deviation from the tier median ranges −22.4%..+3.1% while +`ratio_to_anchor` holds to a median 0.03% and a worst 1.75%. + +**The gate is on ratio coherence, not on anchor drift** — which is the obvious +design and is wrong. Snapshot `a588a69d2759` (EPYC 9V74) sits 22.4% off its +window's anchor and is uniformly 22.4% fast in raw `ns/op` across all 162 of its +benchmarks, agreeing with its window on every ratio to within 0.1%: the host was +fast and the anchor divided that back out, which is what the anchor is for. +Gating on anchor deviation would discard it and two more like it, two of which +are snapshots this baseline is seeded from. + +What does need rejecting is the *mixed* capture — anchor caught the slow tail, +benchmarks did not — where every ratio is uniformly wrong while the raw numbers +look ordinary. That shows up as a whole-snapshot offset in normalized space, +well clear of the 1.75% the corpus exhibits. This approach makes the baseline auditable (provenance is in git history), CI-sourced (no local machine capture noise), and reproducible across runs (idempotent). @@ -293,9 +326,11 @@ the ratio comparison is on shakier ground. The active `docs/perf/baseline.json` is seeded from CI timeline snapshots and gates against the incremental (newest) snapshot for real-time drift tracking: -- **amd64 profiles**: Seeded from the newest snapshot per explicit amd64 machine - key (e.g. AMD EPYC 7763, 9V74, etc.) to survive runner rotation and provide - a coarse gate on amd64 systems where runner noise is <5% within a capture. +- **amd64 profiles**: Seeded from a median over the most recent snapshots per + amd64 machine key (e.g. AMD EPYC 7763, 9V74) to survive runner rotation. The + gate budget is 5%; no null control has been run on these tiers, so the floor + beneath that budget — the gap between two builds that cannot differ — is not + yet measured here. - **arm64/Apple M3**: Preserved from any existing local baseline, allowing M3 developers to gate against a machine-specific baseline without CI noise. @@ -308,7 +343,7 @@ The `captured_at_sha` field in each machine entry points to the incremental SHA, representing the "current" state for `make bench-ratchet check` comparisons. The `bench-ratchet seed-baseline` command (§ [Seeding from CI](#seeding-from-ci)) -selects and merges timeline snapshots from the perf-data branch, making the +reduces a window of timeline snapshots from the perf-data branch, making the baseline reproducible, auditable, and independent of local machine captures. ## Current baseline: amd64-seeded with M3 fallback @@ -316,8 +351,9 @@ baseline reproducible, auditable, and independent of local machine captures. The active `docs/perf/baseline.json` is seeded from CI timeline snapshots via `seed-baseline`, with amd64 as the primary machine tier and the existing local M3 profile (arm64/Apple M3) preserved for local developer gating. This means -`make bench-ratchet` gates against the most recent successful amd64 build in -CI, providing a durable, reproducible baseline free of local machine noise. +`make bench-ratchet` gates against a median of the most recent successful amd64 +builds in CI, providing a durable, reproducible baseline free of local machine +noise. A future release-reference baseline (#597, separate) will backfill v1.8.0 reference snapshots for each machine tier to answer "how do we compare to the