New browser instance per benchmark task

packages/web/benchmark/src/index.ts

@@ -1,10 +1,8 @@
 import { BENCHMARK_TASKS, createServices } from "./tasks";
 import { BenchmarkConfig, BenchmarkResults, QueryResult, SourceResult } from "./types";
+import { DEFAULT_ITERATIONS, DEFAULT_WARMUP_ROUNDS, buildQueryResult } from "./stats";
 import DatabaseServiceWebWorker from "../../src/services/DatabaseServiceWeb/duckdb-worker.worker";
 
-const DEFAULT_ITERATIONS = 5;
-const DEFAULT_WARMUP_ROUNDS = 1;
-
 // Updates the #status element in the benchmark HTML page and mirrors to console.
 // The page can run headlessly in CI (Playwright), so the console log is the
 // only visible progress signal when there is no browser UI to observe.
@@ -14,14 +12,6 @@ function setStatus(msg: string) {
     console.log("[benchmark]", msg);
 }
 
-// Nearest-rank percentile over a pre-sorted array. Used to report p50 and p95
-// across timed iterations — p95 surfaces occasional slow outliers (GC pauses,
-// DuckDB cache misses) that the median would hide.
-function percentile(sorted: number[], p: number): number {
-    const idx = Math.ceil((p / 100) * sorted.length) - 1;
-    return sorted[Math.max(0, idx)];
-}
-
 // Fisher-Yates shuffle — randomizes task order each timed iteration so that a
 // consistently slow task doesn't inflate the times of everything that follows it
 // (DuckDB buffer pool and OS page cache warm up over repeated runs).
@@ -53,7 +43,8 @@ async function benchmarkSource(
     service: DatabaseServiceWebWorker,
     sourceNames: string[],
     iterations: number,
-    warmupRounds: number
+    warmupRounds: number,
+    tasks: typeof BENCHMARK_TASKS
 ): Promise<QueryResult[]> {
     const { annotationSvc, fileSvc } = createServices(service, sourceNames);
 
@@ -64,17 +55,17 @@ async function benchmarkSource(
     // of every task reflect cold-start overhead rather than steady-state cost.
     setStatus(`Warming up ${sourceNames.join(", ")} (${warmupRounds} rounds)...`);
     for (let w = 0; w < warmupRounds; w++) {
-        for (const task of BENCHMARK_TASKS) {
+        for (const task of tasks) {
             service.clearTimings();
             await task.run(annotationSvc, fileSvc);
         }
     }
 
-    const timingsMap = new Map<string, number[]>(BENCHMARK_TASKS.map(({ name }) => [name, []]));
+    const timingsMap = new Map<string, number[]>(tasks.map(({ name }) => [name, []]));
 
     for (let i = 0; i < iterations; i++) {
         setStatus(`Timing ${sourceNames.join(", ")} — iteration ${i + 1}/${iterations}...`);
-        for (const task of shuffle(BENCHMARK_TASKS)) {
+        for (const task of shuffle(tasks)) {
             if (task.resetAnnotationCache) {
                 for (const sourceName of sourceNames) {
                     service.clearAnnotationCache(sourceName);
@@ -94,16 +85,7 @@ async function benchmarkSource(
         }
     }
 
-    return BENCHMARK_TASKS.map(({ name }) => {
-        const timings = [...(timingsMap.get(name) ?? [])].sort((a, b) => a - b);
-        return {
-            name,
-            timings,
-            p50: percentile(timings, 50),
-            p95: percentile(timings, 95),
-            p99: percentile(timings, 99),
-        };
-    });
+    return tasks.map(({ name }) => buildQueryResult(name, timingsMap.get(name) ?? []));
 }
 
 async function main() {
@@ -113,6 +95,16 @@ async function main() {
     }
     const iterations = config.iterations ?? DEFAULT_ITERATIONS;
     const warmupRounds = config.warmupRounds ?? DEFAULT_WARMUP_ROUNDS;
+    const taskFilter = config.taskFilter;
+
+    // When a taskFilter is provided, only run the requested tasks.
+    if (taskFilter) {
+        const validNames = new Set(BENCHMARK_TASKS.map((t) => t.name));
+        const invalid = taskFilter.filter((n) => !validNames.has(n));
+        if (invalid.length) {
+            throw new Error(`Unknown task(s) in taskFilter: ${invalid.join(", ")}`);
+        }
+    }
 
     setStatus("Initializing DuckDB-WASM...");
     const initStart = performance.now();
@@ -149,6 +141,10 @@ async function main() {
         await service.execute('DROP VIEW IF EXISTS "__bff_warmup__"');
     }
 
+    const activeTasks = taskFilter
+        ? BENCHMARK_TASKS.filter((t) => taskFilter.includes(t.name))
+        : BENCHMARK_TASKS;
+
     const sourceResults: SourceResult[] = [];
 
     for (const sources of config.testCases) {
@@ -171,7 +167,13 @@ async function main() {
         const registrationMs = performance.now() - regStart;
 
         const labels = sources.map((source) => source.label);
-        const queries = await benchmarkSource(service, labels, iterations, warmupRounds);
+        const queries = await benchmarkSource(
+            service,
+            labels,
+            iterations,
+            warmupRounds,
+            activeTasks
+        );
         sourceResults.push({ labels, registrationMs, queries });
 
         for (const source of sources) {

packages/web/benchmark/src/stats.ts

@@ -0,0 +1,20 @@
+import { QueryResult } from "./types";
+
+export const DEFAULT_ITERATIONS = 5;
+export const DEFAULT_WARMUP_ROUNDS = 1;
+
+export function percentile(sorted: number[], p: number): number {
+    const idx = Math.ceil((p / 100) * sorted.length) - 1;
+    return sorted[Math.max(0, idx)];
+}
+
+export function buildQueryResult(name: string, rawTimings: number[]): QueryResult {
+    const timings = [...rawTimings].sort((a, b) => a - b);
+    return {
+        name,
+        timings,
+        p50: percentile(timings, 50),
+        p95: percentile(timings, 95),
+        p99: percentile(timings, 99),
+    };
+}

packages/web/benchmark/src/types.ts

@@ -15,6 +15,7 @@ export interface BenchmarkConfig {
     testCases: TestCase[];
     iterations?: number;
     warmupRounds?: number;
+    taskFilter?: string[];
 }
 
 export interface QueryResult {

packages/web/scripts/lib/run-benchmark-page.ts

@@ -22,11 +22,14 @@
 }
 
 import { chromium } from "playwright";
+import type { Page } from "playwright";
 import http from "http";
 import fs from "fs";
 import path from "path";
 import { execSync } from "child_process";
-import { BenchmarkResults, TestCase } from "../../benchmark/src/types";
+import { BenchmarkResults, SourceResult, TestCase } from "../../benchmark/src/types";
+import { BENCHMARK_TASKS } from "../../benchmark/src/tasks";
+import { DEFAULT_ITERATIONS, buildQueryResult } from "../../benchmark/src/stats";
 
 const DIST_DIR = path.join(__dirname, "..", "..", "benchmark", "dist");
 const FIXTURES_DIR = path.join(__dirname, "..", "..", "fixtures");
@@ -167,6 +170,108 @@
     }
 
     const server = await startServer();
+
+    try {
+        if (warmupRounds === 0) {
+            return await runColdStartBenchmarks({ testCases, iterations, channel });
+        } else {
+            // Warmups > 0: all test cases share a single browser instance.
+            return await runInBrowser({ testCases, iterations, warmupRounds, channel });
+        }
+    } finally {
+        await new Promise((res) => server.close(res));
+    }
+}
+
+/**
+ * warmupRounds=0 path: each (testCase, task, iteration) triple gets a fresh
+ * browser so every measurement is a cold start.
+ */
+async function runColdStartBenchmarks({
+    testCases,
+    iterations,
+    channel,
+}: {
+    testCases: TestCase[];
+    iterations?: number;
+    channel?: string;
+}): Promise<BenchmarkResults> {
+    const allTaskNames = BENCHMARK_TASKS.map((task) => task.name);
+    const iterationCount = iterations ?? DEFAULT_ITERATIONS;
+
+    let initTimeMs = 0;
+    const sourceResults: SourceResult[] = [];
+
+    for (const testCase of testCases) {
+        let registrationMs = 0;
+
+        console.log(
+            `[playwright] warmupRounds=0: running ${allTaskNames.length} task(s) × ` +
+                `${iterationCount} iteration(s) in separate browser instances`
+        );
+
+        const timingsMap = new Map<string, number[]>(allTaskNames.map((name) => [name, []]));
+
+        for (const taskName of allTaskNames) {
+            for (let i = 0; i < iterationCount; i++) {
+                console.log(
+                    `[playwright] Launching browser for "${taskName}" ` +
+                        `iteration ${i + 1}/${iterationCount} ` +
+                        `(${testCase.map((source) => source.label).join(", ")})`
+                );
+                const run = await runInBrowser({
+                    testCases: [testCase],
+                    iterations: 1,
+                    warmupRounds: 0,
+                    channel,
+                    taskFilter: [taskName],
+                });
+
+                if (initTimeMs === 0) initTimeMs = run.initTimeMs;
+                const runResult = run.results[0];
+                if (registrationMs === 0) registrationMs = runResult.registrationMs;
+
+                const timing = runResult.queries[0]?.timings[0];
+                if (timing !== undefined) {
+                    timingsMap.get(taskName)!.push(timing);
+                }
+            }
+        }
+
+        sourceResults.push({
+            labels: testCase.map((source) => source.label),
+            registrationMs,
+            queries: allTaskNames.map((name) => buildQueryResult(name, timingsMap.get(name) ?? [])),
+        });
+    }
+
+    return {
+        timestamp: new Date().toISOString(),
+        commit: "unknown",
+        branch: "unknown",
+        initTimeMs,
+        results: sourceResults,
+    };
+}
+
+/**
+ * Launch a single browser, run the benchmark with the given config, and return
+ * the raw BenchmarkResults. Both the warmup>0 path (all test cases) and the
+ * warmup=0 path (one test case + task filter) funnel through here.
+ */
+async function runInBrowser({
+    testCases,
+    iterations,
+    warmupRounds,
+    channel,
+    taskFilter,
+}: {
+    testCases: TestCase[];
+    iterations?: number;
+    warmupRounds?: number;
+    channel?: string;
+    taskFilter?: string[];
+}): Promise<BenchmarkResults> {
     const browser = await chromium.launch({
         channel,
         headless: true,
@@ -188,6 +293,7 @@
                 testCases,
                 iterations,
                 warmupRounds,
+                taskFilter,
             })};`,
         });
 
@@ -203,38 +309,7 @@
         // The browser reads the file lazily via FileReader (BROWSER_FILEREADER protocol),
         // which is identical to how the real app loads files via the file picker —
         // no HTTP range-request overhead, so DuckDB sort performance matches real-user timing.
-        const loaded = new Set();
-        for (const source of testCases.flat()) {
-            if (source.label in loaded) continue; // Don't add duplicate sources
-            const localMatch = source.url.match(
-                new RegExp(`^http://localhost:${PORT}/fixtures/(.+)$`)
-            );
-            if (!localMatch) continue;
-            const fixturePath = path.join(FIXTURES_DIR, localMatch[1]);
-            if (!fs.existsSync(fixturePath)) continue;
-
-            console.log(`[playwright] Injecting ${source.label} via setInputFiles...`);
-            const inputHandle = await page.evaluateHandle(() => {
-                const inp: HTMLInputElement = document.createElement("input");
-                inp.type = "file";
-                document.body.appendChild(inp);
-                return inp;
-            });
-            await inputHandle.setInputFiles(fixturePath);
-            await page.evaluate((label) => {
-                const inputs: NodeListOf<HTMLInputElement> = document.querySelectorAll(
-                    "input[type=file]"
-                );
-                const inp = inputs[inputs.length - 1];
-                window.__pendingLocalFiles = window.__pendingLocalFiles || {};
-                if (!inp.files) {
-                    throw new Error(`Injected file not found for ${label}.`);
-                }
-                window.__pendingLocalFiles[label] = inp.files[0];
-                inp.remove();
-            }, source.label);
-            loaded.add(source.label);
-        }
+        await injectFixtures(page, testCases);
 
         // Signal the benchmark to proceed with injected File objects
         await page.evaluate(() => {
@@ -257,7 +332,39 @@
         return await page.evaluate(() => window.__benchmarkResults);
     } finally {
         await browser.close();
-        await new Promise((res) => server.close(res));
+    }
+}
+
+async function injectFixtures(page: Page, testCases: TestCase[]) {
+    const loaded = new Set<string>();
+    for (const source of testCases.flat()) {
+        if (loaded.has(source.label)) continue;
+        const localMatch = source.url.match(new RegExp(`^http://localhost:${PORT}/fixtures/(.+)$`));
+        if (!localMatch) continue;
+        const fixturePath = path.join(FIXTURES_DIR, localMatch[1]);
+        if (!fs.existsSync(fixturePath)) continue;
+
+        console.log(`[playwright] Injecting ${source.label} via setInputFiles...`);
+        const inputHandle = await page.evaluateHandle(() => {
+            const inp: HTMLInputElement = document.createElement("input");
+            inp.type = "file";
+            document.body.appendChild(inp);
+            return inp;
+        });
+        await inputHandle.setInputFiles(fixturePath);
+        await page.evaluate((label) => {
+            const inputs: NodeListOf<HTMLInputElement> = document.querySelectorAll(
+                "input[type=file]"
+            );
+            const inp = inputs[inputs.length - 1];
+            window.__pendingLocalFiles = window.__pendingLocalFiles || {};
+            if (!inp.files) {
+                throw new Error(`Injected file not found for ${label}.`);
+            }
+            window.__pendingLocalFiles[label] = inp.files[0];
+            inp.remove();
+        }, source.label);
+        loaded.add(source.label);
     }
 }