diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4a3d8e957..ce1d2842b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -36,8 +36,8 @@ jobs: # hardware variance between VMs would mask the small regressions we actually care about. # Running back-to-back on the same VM ensures both measurements share the same hardware # baseline, so deltas reflect code differences only. - # 180 minutes: fixture download + full task suite (including change_grouping on 10m) × 2 branches - timeout-minutes: 180 + # 360 minutes is the maximum timeout + timeout-minutes: 360 steps: - uses: actions/checkout@v4 @@ -51,7 +51,7 @@ jobs: uses: actions/cache@v4 with: path: packages/web/fixtures - key: benchmark-fixtures-v1 + key: benchmark-fixtures-v1.2 - name: Download benchmark fixtures if: steps.fixture-cache.outputs.cache-hit != 'true' @@ -61,6 +61,8 @@ jobs: curl -fL "$BASE/synthetic-100k.parquet" -o packages/web/fixtures/synthetic-100k.parquet curl -fL "$BASE/synthetic-1m.parquet" -o packages/web/fixtures/synthetic-1m.parquet curl -fL "$BASE/synthetic-10m.parquet" -o packages/web/fixtures/synthetic-10m.parquet + curl -fL "$BASE/synthetic-10m-copy.parquet" -o packages/web/fixtures/synthetic-10m-copy.parquet + curl -fL "$BASE/synthetic-20m.parquet" -o packages/web/fixtures/synthetic-20m.parquet - uses: actions/setup-node@v4 with: @@ -75,7 +77,7 @@ jobs: working-directory: packages/web - name: Run benchmark (${{ github.event.inputs.compare_branch }}) - run: node scripts/run-regression.js --iterations ${{ github.event.inputs.iterations }} --warmup ${{ github.event.inputs.warmup }} + run: npm run benchmark:regression -- --iterations ${{ github.event.inputs.iterations }} --warmup ${{ github.event.inputs.warmup }} working-directory: packages/web env: BENCHMARK_BRANCH: ${{ github.event.inputs.compare_branch }} @@ -92,15 +94,16 @@ jobs: run: npm ci - name: Run benchmark (${{ github.event.inputs.base_branch }}) - run: node scripts/run-regression.js --skip-build --iterations ${{ github.event.inputs.iterations }} --warmup ${{ github.event.inputs.warmup }} + run: npm run benchmark:regression -- --skip-build --iterations ${{ github.event.inputs.iterations }} --warmup ${{ github.event.inputs.warmup }} working-directory: packages/web env: BENCHMARK_BRANCH: ${{ github.event.inputs.base_branch }} - name: Generate comparison run: | - BASE_FILE=$(ls packages/web/benchmark-results-*.json | head -1) - node packages/web/scripts/compare-results.js "$BASE_FILE" /tmp/benchmark-compare.json >> "$GITHUB_STEP_SUMMARY" + BASE_FILE=$(ls benchmark-results-*.json | head -1) + npm run benchmark:compare -- "$BASE_FILE" /tmp/benchmark-compare.json >> "$GITHUB_STEP_SUMMARY" + working-directory: packages/web - name: Upload results if: always() diff --git a/dev-docs/07-query-benchmarking.md b/dev-docs/07-query-benchmarking.md index 66c79d513..5bd7cb752 100644 --- a/dev-docs/07-query-benchmarking.md +++ b/dev-docs/07-query-benchmarking.md @@ -25,6 +25,8 @@ mkdir -p packages/web/fixtures curl -fL "$BASE/synthetic-100k.parquet" -o packages/web/fixtures/synthetic-100k.parquet curl -fL "$BASE/synthetic-1m.parquet" -o packages/web/fixtures/synthetic-1m.parquet curl -fL "$BASE/synthetic-10m.parquet" -o packages/web/fixtures/synthetic-10m.parquet +cp packages/web/fixtures/synthetic-10m.parquet packages/web/fixtures/synthetic-10m-copy.parquet +curl -fL "$BASE/synthetic-20m.parquet" -o packages/web/fixtures/synthetic-20m.parquet ``` **Run against local fixtures** @@ -62,7 +64,7 @@ This prints a Markdown table with p50 deltas and regression/improvement badges ( | Flag | Description | |---|---| | `--local` | Use fixtures from `packages/web/fixtures/` instead of S3 URLs | -| `--scale 100k\|1m\|10m` | Run a single fixture size | +| `--scale 100k\|1m\|10m\|10m+10m\|20m` | Run a single fixture size | | `--full` | Run all scales with both cloud and local sources side-by-side | | `--iterations N` | Timed iterations per task (default 5) | | `--warmup N` | Warmup rounds before timing (default 1) | @@ -82,10 +84,10 @@ Both branches run on the same machine to eliminate hardware variance — a ~15% The workflow: 1. Checks out the compare branch and downloads fixtures from S3 (cached by version) -2. Runs `run-regression.js` → writes `benchmark-results-.json` +2. Runs `run-regression.ts` → writes `benchmark-results-.json` 3. Checks out the base branch (without wiping fixtures) -4. Runs `run-regression.js` → writes `benchmark-results-.json` -5. Runs `compare-results.js` → posts the Markdown table to the step summary +4. Runs `run-regression.ts` → writes `benchmark-results-.json` +5. Runs `compare-results.ts` → posts the Markdown table to the step summary --- diff --git a/package-lock.json b/package-lock.json index ff3c50ca1..3ba0b627c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -96,6 +96,7 @@ "resize-observer": "1.0.x", "rimraf": "3.0.x", "sinon": "12.x", + "tsx": "^4.21.0", "typescript": "4.9.x" }, "engines": { @@ -2712,6 +2713,448 @@ "node": ">= 10.0.0" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.9.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", @@ -9783,6 +10226,48 @@ "license": "MIT", "optional": true }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -11338,6 +11823,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-tsconfig": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", + "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -18053,6 +18551,16 @@ "node": ">=4" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/responselike": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.1.tgz", @@ -20006,6 +20514,26 @@ "dev": true, "license": "0BSD" }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index ae4252442..cd4d32e31 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "resize-observer": "1.0.x", "rimraf": "3.0.x", "sinon": "12.x", + "tsx": "4.21.x", "typescript": "4.9.x" }, "dependencies": { diff --git a/packages/web/benchmark/src/index.ts b/packages/web/benchmark/src/index.ts index baba1d835..02d79893e 100644 --- a/packages/web/benchmark/src/index.ts +++ b/packages/web/benchmark/src/index.ts @@ -51,18 +51,18 @@ function shuffle(arr: T[]): T[] { */ async function benchmarkSource( service: DatabaseServiceWebWorker, - sourceName: string, + sourceNames: string[], iterations: number, warmupRounds: number ): Promise { - const { annotationSvc, fileSvc } = createServices(service, sourceName); + const { annotationSvc, fileSvc } = createServices(service, sourceNames); service.enableQueryTiming(); // Warmup ensures DuckDB's buffer pool, query planner, and V8 JIT are in a // stable state before timing begins. Without it, the first few iterations // of every task reflect cold-start overhead rather than steady-state cost. - setStatus(`Warming up ${sourceName} (${warmupRounds} rounds)...`); + setStatus(`Warming up ${sourceNames.join(", ")} (${warmupRounds} rounds)...`); for (let w = 0; w < warmupRounds; w++) { for (const task of BENCHMARK_TASKS) { service.clearTimings(); @@ -73,10 +73,12 @@ async function benchmarkSource( const timingsMap = new Map(BENCHMARK_TASKS.map(({ name }) => [name, []])); for (let i = 0; i < iterations; i++) { - setStatus(`Timing ${sourceName} — iteration ${i + 1}/${iterations}...`); + setStatus(`Timing ${sourceNames.join(", ")} — iteration ${i + 1}/${iterations}...`); for (const task of shuffle(BENCHMARK_TASKS)) { if (task.resetAnnotationCache) { - service.clearAnnotationCache(sourceName); + for (const sourceName of sourceNames) { + service.clearAnnotationCache(sourceName); + } } const timings = timingsMap.get(task.name) ?? []; timingsMap.set(task.name, timings); @@ -106,7 +108,7 @@ async function benchmarkSource( async function main() { const config: BenchmarkConfig = (window as any).__benchmarkConfig; - if (!config?.sources?.length) { + if (!config?.testCases?.length) { throw new Error("No benchmark config found. Runner must inject window.__benchmarkConfig."); } const iterations = config.iterations ?? DEFAULT_ITERATIONS; @@ -137,8 +139,8 @@ async function main() { // Absorb DuckDB's one-time parquet cold-start cost (scanner JIT, VFS setup, // buffer pool init) before timing any real source registrations. Without this, // the first source always shows inflated registration time regardless of file size. - if (config.sources.length > 0) { - const warmup = config.sources[0]; + if (config.testCases.length > 0) { + const warmup = config.testCases[0][0]; const warmupFile = localFiles[warmup.label]; await service.prepareDataSources( [{ name: "__bff_warmup__", type: "parquet", uri: warmupFile ?? warmup.url }], @@ -147,28 +149,34 @@ async function main() { await service.execute('DROP VIEW IF EXISTS "__bff_warmup__"'); } - const sources: SourceResult[] = []; - - for (const source of config.sources) { - setStatus(`Registering ${source.label} (${source.url})...`); + const sourceResults: SourceResult[] = []; + for (const sources of config.testCases) { const regStart = performance.now(); - const localFile = localFiles[source.label]; - if (!localFile) { - console.warn( - `[benchmark] No local file for ${source.label} — falling back to HTTP reads; timings will differ from local-file runs` - ); - } + await service.prepareDataSources( - [{ name: source.label, type: "parquet", uri: localFile ?? source.url }], + sources.map((source) => { + setStatus(`Registering ${source.label} (${source.url})...`); + const localFile = localFiles[source.label]; + if (!localFile) { + console.warn( + `[benchmark] No local file for ${source.label} — falling back to HTTP reads; timings will differ from local-file runs` + ); + } + return { name: source.label, type: "parquet", uri: localFile ?? source.url }; + }), /* skipNormalization */ true ); + const registrationMs = performance.now() - regStart; - const queries = await benchmarkSource(service, source.label, iterations, warmupRounds); - sources.push({ label: source.label, registrationMs, queries }); + const labels = sources.map((source) => source.label); + const queries = await benchmarkSource(service, labels, iterations, warmupRounds); + sourceResults.push({ labels, registrationMs, queries }); - await service.execute(`DROP VIEW IF EXISTS "${source.label}"`); + for (const source of sources) { + await service.deleteDataSourceWrapper(source.label); + } } setStatus("Done."); @@ -178,7 +186,7 @@ async function main() { commit: "unknown", branch: "unknown", initTimeMs, - sources, + results: sourceResults, }; (window as any).__benchmarkResults = results; diff --git a/packages/web/benchmark/src/tasks.ts b/packages/web/benchmark/src/tasks.ts index 53c8eb96c..2ef603f21 100644 --- a/packages/web/benchmark/src/tasks.ts +++ b/packages/web/benchmark/src/tasks.ts @@ -163,15 +163,15 @@ export const BENCHMARK_TASKS: BenchmarkTask[] = [ /** Create service instances wrapping the given worker for one data source. */ export function createServices( db: DatabaseServiceWebWorker, - sourceName: string + sourceNames: string[] ): { annotationSvc: DatabaseAnnotationService; fileSvc: DatabaseFileService } { const annotationSvc = new DatabaseAnnotationService({ databaseService: db, - dataSourceNames: [sourceName], + dataSourceNames: sourceNames, }); const fileSvc = new DatabaseFileService({ databaseService: db, - dataSourceNames: [sourceName], + dataSourceNames: sourceNames, downloadService: new FileDownloadServiceNoop(), }); return { annotationSvc, fileSvc }; diff --git a/packages/web/benchmark/src/types.ts b/packages/web/benchmark/src/types.ts index 1df530db1..3f1cb6f25 100644 --- a/packages/web/benchmark/src/types.ts +++ b/packages/web/benchmark/src/types.ts @@ -8,9 +8,11 @@ export interface ParquetSource { label: string; } +export type TestCase = ParquetSource[]; + /** Injected as window.__benchmarkConfig before the page loads. */ export interface BenchmarkConfig { - sources: ParquetSource[]; + testCases: TestCase[]; iterations?: number; warmupRounds?: number; } @@ -24,7 +26,7 @@ export interface QueryResult { } export interface SourceResult { - label: string; + labels: string[]; registrationMs: number; queries: QueryResult[]; } @@ -34,5 +36,5 @@ export interface BenchmarkResults { commit: string; branch: string; initTimeMs: number; - sources: SourceResult[]; + results: SourceResult[]; } diff --git a/packages/web/package.json b/packages/web/package.json index a170f3b24..9e1998f5f 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -3,9 +3,9 @@ "description": "A web application for organizing that data, and provide simple hooks for incorporating that data into both programmatic and non-programmatic workflows", "main": "src/index.tsx", "scripts": { - "benchmark": "node scripts/run-local.js", - "benchmark:regression": "node scripts/run-regression.js", - "benchmark:compare": "node scripts/compare-results.js", + "benchmark": "tsx scripts/run-local.ts", + "benchmark:regression": "tsx scripts/run-regression.ts", + "benchmark:compare": "tsx scripts/compare-results.ts", "benchmark:summary": "node scripts/summarize-results.js", "build": "webpack --config ./webpack/webpack.config.js --env production", "clean": "git clean -Xfd -e \"!node_modules\"", diff --git a/packages/web/scripts/compare-results.js b/packages/web/scripts/compare-results.ts similarity index 82% rename from packages/web/scripts/compare-results.js rename to packages/web/scripts/compare-results.ts index dff4766ed..f95b4ec12 100644 --- a/packages/web/scripts/compare-results.js +++ b/packages/web/scripts/compare-results.ts @@ -3,7 +3,9 @@ "use strict"; -const fs = require("fs"); +import { BenchmarkResults } from "../benchmark/src/types"; + +import fs from "fs"; const REGRESSION_WARN_PCT = 25; // ≥25% slower → ⚠️ const REGRESSION_SEVERE_PCT = 50; // ≥50% slower → ❌ @@ -12,17 +14,17 @@ const IMPROVEMENT_PCT = 25; // ≥25% faster → ✅ // deltas on fast queries are dominated by noise rather than real regressions. const BADGE_MIN_MS = 500; -function fmt(ms) { +function fmt(ms: number | null | undefined) { if (ms === undefined || ms === null) return "—"; return ms < 10 ? `${ms.toFixed(2)}ms` : `${ms.toFixed(1)}ms`; } -function pctDelta(base, pr) { +function pctDelta(base: number, pr: number) { if (!base) return null; return ((pr - base) / base) * 100; } -function deltaBadge(base, pr) { +function deltaBadge(base: number, pr: number) { const delta = pctDelta(base, pr); if (delta === null) return "N/A"; const sign = delta >= 0 ? "+" : ""; @@ -41,21 +43,24 @@ if (!baseFile || !prFile) { process.exit(1); } -const base = JSON.parse(fs.readFileSync(baseFile, "utf8")); -const pr = JSON.parse(fs.readFileSync(prFile, "utf8")); +const base: BenchmarkResults = JSON.parse(fs.readFileSync(baseFile, "utf8")); +const pr: BenchmarkResults = JSON.parse(fs.readFileSync(prFile, "utf8")); -const baseSources = new Map(base.sources.map((s) => [s.label, s])); -const prSources = new Map(pr.sources.map((s) => [s.label, s])); +const baseSources = new Map(base.results.map((s) => [s.labels.join(", "), s])); +const prSources = new Map(pr.results.map((s) => [s.labels.join(", "), s])); // PR result order is authoritative; base may have fewer sources. const allLabels = [ - ...new Set([...pr.sources.map((s) => s.label), ...base.sources.map((s) => s.label)]), + ...new Set([ + ...pr.results.map((s) => s.labels.join(", ")), + ...base.results.map((s) => s.labels.join(", ")), + ]), ]; const allQueryNames = [ ...new Set([ - ...pr.sources.flatMap((s) => s.queries.map((q) => q.name)), - ...base.sources.flatMap((s) => s.queries.map((q) => q.name)), + ...pr.results.flatMap((s) => s.queries.map((q) => q.name)), + ...base.results.flatMap((s) => s.queries.map((q) => q.name)), ]), ]; @@ -66,12 +71,15 @@ for (const qName of allQueryNames) { const baseQ = baseSources.get(label)?.queries.find((q) => q.name === qName); const prQ = prSources.get(label)?.queries.find((q) => q.name === qName); if (baseQ && prQ) { - allDeltas.push({ - label: `\`${qName}\` @ ${label}`, - delta: pctDelta(baseQ.p50, prQ.p50), - baseP50: baseQ.p50, - prP50: prQ.p50, - }); + const delta = pctDelta(baseQ.p50, prQ.p50); + if (delta !== null) { + allDeltas.push({ + label: `\`${qName}\` @ ${label}`, + delta, + baseP50: baseQ.p50, + prP50: prQ.p50, + }); + } } } } @@ -190,7 +198,7 @@ if (regressions.length === 0 && improvements.length === 0) { } } -const iters = pr.sources[0]?.queries[0]?.timings?.length ?? "?"; +const iters = pr.results[0]?.queries[0]?.timings?.length ?? "?"; lines.push( `_Benchmarks run in headless Chromium with DuckDB-WASM. ` + `${iters} iterations per query. ` + diff --git a/packages/web/scripts/lib/run-benchmark-page.js b/packages/web/scripts/lib/run-benchmark-page.ts similarity index 79% rename from packages/web/scripts/lib/run-benchmark-page.js rename to packages/web/scripts/lib/run-benchmark-page.ts index c8ead326d..62134d69a 100644 --- a/packages/web/scripts/lib/run-benchmark-page.js +++ b/packages/web/scripts/lib/run-benchmark-page.ts @@ -6,13 +6,27 @@ * BenchmarkConfig into the page, and returns the BenchmarkResults. */ -"use strict"; +// This script uses the global window object to communicate with the playwright +// controller. The type declaration here adds the necessary variables to the +// window object's type, Window. +declare global { + interface Window { + __localFilesRequested: boolean; + __pendingLocalFiles: any; + __resolveLocalFiles: ( + value: Record | PromiseLike> + ) => void; + __benchmarkResults: BenchmarkResults; + __benchmarkError: string; + } +} -const { chromium } = require("playwright"); -const http = require("http"); -const fs = require("fs"); -const path = require("path"); -const { execSync } = require("child_process"); +import { chromium } from "playwright"; +import http from "http"; +import fs from "fs"; +import path from "path"; +import { execSync } from "child_process"; +import { BenchmarkResults, TestCase } from "../../benchmark/src/types"; const DIST_DIR = path.join(__dirname, "..", "..", "benchmark", "dist"); const FIXTURES_DIR = path.join(__dirname, "..", "..", "fixtures"); @@ -30,17 +44,20 @@ const MIME = { ".parquet": "application/octet-stream", }; -function mimeFor(filePath) { +function mimeFor(filePath: string) { for (const [ext, type] of Object.entries(MIME)) { if (filePath.endsWith(ext)) return type; } return "application/octet-stream"; } -function startServer() { +function startServer(): Promise< + http.Server +> { return new Promise((resolve, reject) => { const server = http.createServer((req, res) => { - const relPath = req.url === "/" ? "/index.html" : req.url.split("?")[0]; + const relPath = + req.url === "/" || req.url === undefined ? "/index.html" : req.url.split("?")[0]; // Serve fixture files from /fixtures/ — fallback path if file injection fails. const fixtureMatch = relPath.match(/^\/fixtures\/(.+)$/); @@ -117,7 +134,7 @@ function buildBenchmark() { * Run the benchmark page with the given config and return the BenchmarkResults. * * @param {object} options - * @param {{ url: string, label: string }[]} options.sources Parquet sources to benchmark. + * @param {{ url: string, label: string }[][]} options.testCases Parquet sources to benchmark. * @param {boolean} [options.skipBuild=false] Skip webpack build. * @param {number} [options.iterations] Override timed iteration count. * @param {number} [options.warmupRounds] Override warmup round count. @@ -128,7 +145,19 @@ function buildBenchmark() { * Chromium (default; required for CI). * @returns {Promise} Raw BenchmarkResults from the page. */ -async function runBenchmarkPage({ sources, skipBuild = false, iterations, warmupRounds, channel }) { +export async function runBenchmarkPage({ + testCases, + skipBuild = false, + iterations, + warmupRounds, + channel, +}: { + testCases: TestCase[]; + skipBuild?: boolean; + iterations?: number; + warmupRounds?: number; + channel?: string; +}): Promise { if (!skipBuild) buildBenchmark(); if (!fs.existsSync(path.join(DIST_DIR, "index.html"))) { @@ -137,11 +166,11 @@ async function runBenchmarkPage({ sources, skipBuild = false, iterations, warmup ); } - const launchOptions = { headless: true }; - if (channel) launchOptions.channel = channel; - const server = await startServer(); - const browser = await chromium.launch(launchOptions); + const browser = await chromium.launch({ + channel, + headless: true, + }); try { const context = await browser.newContext(); @@ -156,13 +185,13 @@ async function runBenchmarkPage({ sources, skipBuild = false, iterations, warmup // synchronously on startup — no callback handshake needed. await page.addInitScript({ content: `window.__benchmarkConfig = ${JSON.stringify({ - sources, + testCases, iterations, warmupRounds, })};`, }); - console.log(`[playwright] Starting benchmark (${sources.length} source(s))...`); + console.log(`[playwright] Starting benchmark (${testCases.length} test case(s))...`); await page.goto(`http://localhost:${PORT}/`, { waitUntil: "domcontentloaded" }); // Wait for the benchmark to signal it's ready for file injection @@ -174,7 +203,9 @@ async function runBenchmarkPage({ sources, skipBuild = false, iterations, warmup // The browser reads the file lazily via FileReader (BROWSER_FILEREADER protocol), // which is identical to how the real app loads files via the file picker — // no HTTP range-request overhead, so DuckDB sort performance matches real-user timing. - for (const source of sources) { + const loaded = new Set(); + for (const source of testCases.flat()) { + if (source.label in loaded) continue; // Don't add duplicate sources const localMatch = source.url.match( new RegExp(`^http://localhost:${PORT}/fixtures/(.+)$`) ); @@ -184,19 +215,25 @@ async function runBenchmarkPage({ sources, skipBuild = false, iterations, warmup console.log(`[playwright] Injecting ${source.label} via setInputFiles...`); const inputHandle = await page.evaluateHandle(() => { - const inp = document.createElement("input"); + const inp: HTMLInputElement = document.createElement("input"); inp.type = "file"; document.body.appendChild(inp); return inp; }); await inputHandle.setInputFiles(fixturePath); await page.evaluate((label) => { - const inputs = document.querySelectorAll("input[type=file]"); + const inputs: NodeListOf = document.querySelectorAll( + "input[type=file]" + ); const inp = inputs[inputs.length - 1]; window.__pendingLocalFiles = window.__pendingLocalFiles || {}; + if (!inp.files) { + throw new Error(`Injected file not found for ${label}.`); + } window.__pendingLocalFiles[label] = inp.files[0]; inp.remove(); }, source.label); + loaded.add(source.label); } // Signal the benchmark to proceed with injected File objects diff --git a/packages/web/scripts/run-local.js b/packages/web/scripts/run-local.js deleted file mode 100644 index e2c1b5ffe..000000000 --- a/packages/web/scripts/run-local.js +++ /dev/null @@ -1,150 +0,0 @@ -// Local benchmark runner for developer machines. Supports cloud (S3/https) and local -// fixtures, single scale or all scales, and side-by-side cloud vs local comparison (--full). - -"use strict"; - -const path = require("path"); -const fs = require("fs"); -const { execSync } = require("child_process"); -const { runBenchmarkPage } = require("./lib/run-benchmark-page"); - -const LOCAL_FIXTURE_MAP = { - "100k": "http://localhost:18765/fixtures/synthetic-100k.parquet", - "1m": "http://localhost:18765/fixtures/synthetic-1m.parquet", - "10m": "http://localhost:18765/fixtures/synthetic-10m.parquet", -}; - -const REMOTE_URL_MAP = { - "100k": - process.env.BENCHMARK_REAL_100K_URL ?? - "https://staging-biofile-finder-datasets.s3.us-west-2.amazonaws.com/benchmark-fixtures/v1/synthetic-100k.parquet", - "1m": - process.env.BENCHMARK_REAL_1M_URL ?? - "https://staging-biofile-finder-datasets.s3.us-west-2.amazonaws.com/benchmark-fixtures/v1/synthetic-1m.parquet", - "10m": - process.env.BENCHMARK_REAL_10M_URL ?? - "https://staging-biofile-finder-datasets.s3.us-west-2.amazonaws.com/benchmark-fixtures/v1/synthetic-10m.parquet", -}; - -const useLocal = process.argv.includes("--local"); -const useFull = process.argv.includes("--full"); - -const scaleArg = (() => { - const idx = process.argv.indexOf("--scale"); - return idx !== -1 ? process.argv[idx + 1] : null; -})(); - -const URL_MAP = useLocal ? LOCAL_FIXTURE_MAP : REMOTE_URL_MAP; - -if (scaleArg && !URL_MAP[scaleArg] && !useFull) { - console.error( - `Error: --scale "${scaleArg}" is not a valid scale. Choose from: ${Object.keys( - URL_MAP - ).join(", ")}` - ); - process.exit(1); -} - -let sources; -if (useFull) { - const missingUrls = Object.entries(REMOTE_URL_MAP) - .filter(([, url]) => !url) - .map(([label]) => ` BENCHMARK_REAL_${label.toUpperCase()}_URL`); - if (missingUrls.length > 0) { - console.error( - `Error: --full requires all three cloud URLs to be set:\n${missingUrls.join("\n")}` - ); - process.exit(1); - } - // Interleave cloud and local sources per scale for easy side-by-side reading - sources = Object.keys(REMOTE_URL_MAP).flatMap((label) => [ - { label: `${label}-cloud`, url: REMOTE_URL_MAP[label] }, - { label: `${label}-local`, url: LOCAL_FIXTURE_MAP[label] }, - ]); -} else { - sources = Object.entries(URL_MAP) - .filter(([label, url]) => Boolean(url) && (!scaleArg || label === scaleArg)) - .map(([label, url]) => ({ label, url })); -} - -if (sources.length === 0) { - console.error( - "No real parquet URLs provided.\n" + - "Set one or more of:\n" + - " BENCHMARK_REAL_100K_URL\n" + - " BENCHMARK_REAL_1M_URL\n" + - " BENCHMARK_REAL_10M_URL\n" + - "Or use --local to serve fixtures from packages/web/fixtures/.\n" + - "Or use --full to run cloud + local for all scales." - ); - process.exit(1); -} - -function getArgValue(flag) { - const idx = process.argv.indexOf(flag); - return idx !== -1 ? parseInt(process.argv[idx + 1], 10) : undefined; -} - -async function main() { - const skipBuild = process.argv.includes("--skip-build"); - const useChromium = process.argv.includes("--chromium"); - const channel = useChromium ? undefined : "chrome"; - const iterations = getArgValue("--iterations"); - const warmup = getArgValue("--warmup"); - - console.log(`[local] Running against ${sources.length} real parquet source(s):`); - for (const { label, url } of sources) { - console.log(` ${label}: ${url}`); - } - if (iterations) console.log(`[local] Iterations: ${iterations}`); - if (warmup !== undefined) console.log(`[local] Warmup rounds: ${warmup}`); - console.log( - `[local] Browser: ${ - channel ? `system Chrome (channel: ${channel})` : "Playwright bundled Chromium" - }` - ); - - const rawResults = await runBenchmarkPage({ - sources, - skipBuild, - iterations, - warmupRounds: warmup, - channel, - }); - - const branch = getBranch(); - const results = { - ...rawResults, - commit: process.env.GITHUB_SHA ?? getCommit(), - branch, - }; - - const outFile = path.join(__dirname, "..", "benchmark-results-local.json"); - fs.writeFileSync(outFile, JSON.stringify(results, null, 2)); - console.log(`\n[local] Results written to ${path.relative(process.cwd(), outFile)}`); - - execSync(`node ${path.join(__dirname, "summarize-results.js")} "${outFile}"`, { - stdio: "inherit", - }); -} - -function getBranch() { - try { - return execSync("git rev-parse --abbrev-ref HEAD", { stdio: "pipe" }).toString().trim(); - } catch { - return "unknown"; - } -} - -function getCommit() { - try { - return execSync("git rev-parse --short HEAD", { stdio: "pipe" }).toString().trim(); - } catch { - return "unknown"; - } -} - -main().catch((err) => { - console.error("[fatal]", err.message); - process.exit(1); -}); diff --git a/packages/web/scripts/run-local.ts b/packages/web/scripts/run-local.ts new file mode 100644 index 000000000..343f7f723 --- /dev/null +++ b/packages/web/scripts/run-local.ts @@ -0,0 +1,161 @@ +// Local benchmark runner for developer machines. Supports cloud (S3/https) and local +// fixtures, single scale or all scales, and side-by-side cloud vs local comparison (--full). + +import { ParquetSource } from "../benchmark/src/types"; + +import path from "path"; +import fs from "fs"; +import { execSync } from "child_process"; +import { runBenchmarkPage } from "./lib/run-benchmark-page"; + +const LOCAL_BASE = "http://localhost:18765/fixtures/synthetic"; +const REMOTE_BASE = + "https://staging-biofile-finder-datasets.s3.us-west-2.amazonaws.com/benchmark-fixtures/v1/synthetic"; + +type FileIdentifier = "100k" | "1m" | "10m" | "10m-copy" | "20m"; +const FILE_TO_ENV = { + "100k": "BENCHMARK_REAL_100K_URL", + "1m": "BENCHMARK_REAL_1M_URL", + "10m": "BENCHMARK_REAL_10M_URL", + "10m-copy": "BENCHMARK_REAL_10M_2_URL", + "20m": "BENCHMARK_REAL_20M_URL", +}; + +type ScaleIdentifier = "100k" | "1m" | "10m" | "10m+10m" | "20m"; +const TEST_CASES_MAP = { + "100k": ["100k"] as FileIdentifier[], + "1m": ["1m"] as FileIdentifier[], + "10m": ["10m"] as FileIdentifier[], + "10m+10m": ["10m", "10m-copy"] as FileIdentifier[], + "20m": ["20m"] as FileIdentifier[], +}; + +function validateScaleArg(scale: string): asserts scale is ScaleIdentifier { + if (!(Object.keys(TEST_CASES_MAP).indexOf(scale) !== -1)) { + throw new Error(); + } +} + +function getURL(partialFileName: FileIdentifier, useLocal: boolean) { + if (!(partialFileName in FILE_TO_ENV)) { + throw new Error( + `${partialFileName} not recognized. Choose from ${Object.keys(FILE_TO_ENV)}` + ); + } + if (useLocal) { + return `${LOCAL_BASE}-${partialFileName}.parquet`; + } else { + return ( + process.env[FILE_TO_ENV[partialFileName]] ?? `${REMOTE_BASE}-${partialFileName}.parquet` + ); + } +} + +function getSources( + partialFileNames: FileIdentifier[], + useLocal: boolean, + addSuffix?: boolean +): ParquetSource[] { + const suffix = useLocal ? "local" : "cloud"; + return partialFileNames.map((file) => { + return { + label: addSuffix ? `${file}-${suffix}` : file, + url: getURL(file, useLocal), + }; + }); +} + +function getArgValue(flag: string) { + const idx = process.argv.indexOf(flag); + return idx !== -1 ? parseInt(process.argv[idx + 1], 10) : undefined; +} + +function parseArgs() { + const scaleIdx = process.argv.indexOf("--scale"); + const scaleArg = scaleIdx !== -1 ? process.argv[scaleIdx + 1] : null; + + return { + useLocal: process.argv.includes("--local"), + useFull: process.argv.includes("--full"), + scaleArg, + skipBuild: process.argv.includes("--skip-build"), + useChromium: process.argv.includes("--chromium"), + iterations: getArgValue("--iterations"), + warmup: getArgValue("--warmup"), + }; +} + +async function main() { + const { useLocal, useFull, scaleArg, skipBuild, useChromium, iterations, warmup } = parseArgs(); + + let testCases: ParquetSource[][]; + if (useFull) { + testCases = []; + Object.values(TEST_CASES_MAP).forEach((files) => { + testCases.push(getSources(files, false)); + testCases.push(getSources(files, true)); + }); + } else if (scaleArg) { + validateScaleArg(scaleArg); + testCases = [getSources(TEST_CASES_MAP[scaleArg], useLocal)]; + } else { + testCases = Object.values(TEST_CASES_MAP).map((files) => { + return getSources(files, useLocal); + }); + } + + const channel = useChromium ? undefined : "chrome"; + + console.log(`[local] Running against ${testCases.length} test case(s):`); + for (const testCase of testCases) { + console.log(testCase); + } + if (iterations) console.log(`[local] Iterations: ${iterations}`); + if (warmup !== undefined) console.log(`[local] Warmup rounds: ${warmup}`); + console.log( + `[local] Browser: ${ + channel ? `system Chrome (channel: ${channel})` : "Playwright bundled Chromium" + }` + ); + + const rawResults = await runBenchmarkPage({ + testCases, + skipBuild, + iterations, + warmupRounds: warmup, + channel, + }); + + const branch = getBranch(); + const results = { + ...rawResults, + commit: process.env.GITHUB_SHA ?? getCommit(), + branch, + }; + + const outFile = path.join(__dirname, "..", "benchmark-results-local.json"); + fs.writeFileSync(outFile, JSON.stringify(results, null, 2)); + console.log(`\n[local] Results written to ${path.relative(process.cwd(), outFile)}`); + + execSync(`node ${path.join(__dirname, "summarize-results.js")} "${outFile}"`, { + stdio: "inherit", + }); +} + +function getBranch() { + try { + return execSync("git rev-parse --abbrev-ref HEAD", { stdio: "pipe" }).toString().trim(); + } catch { + return "unknown"; + } +} + +function getCommit() { + try { + return execSync("git rev-parse --short HEAD", { stdio: "pipe" }).toString().trim(); + } catch { + return "unknown"; + } +} + +main(); diff --git a/packages/web/scripts/run-regression.js b/packages/web/scripts/run-regression.ts similarity index 53% rename from packages/web/scripts/run-regression.js rename to packages/web/scripts/run-regression.ts index 617310ea3..760e4ee12 100644 --- a/packages/web/scripts/run-regression.js +++ b/packages/web/scripts/run-regression.ts @@ -1,36 +1,68 @@ // CI regression runner — benchmarks one branch against local fixtures and writes // benchmark-results-.json. Called once per branch by benchmark.yml. -"use strict"; +import { ParquetSource } from "../benchmark/src/types"; -const path = require("path"); -const fs = require("fs"); -const { execSync } = require("child_process"); -const { runBenchmarkPage } = require("./lib/run-benchmark-page"); +import path from "path"; +import fs from "fs"; +import { execSync } from "child_process"; +import { runBenchmarkPage } from "./lib/run-benchmark-page"; const FIXTURES_DIR = path.join(__dirname, "..", "fixtures"); -const SCALES = ["100k", "1m", "10m"]; -const LOCAL_FIXTURE_MAP = { - "100k": "http://localhost:18765/fixtures/synthetic-100k.parquet", - "1m": "http://localhost:18765/fixtures/synthetic-1m.parquet", - "10m": "http://localhost:18765/fixtures/synthetic-10m.parquet", -}; +const TEST_CASES: ParquetSource[][] = [ + [ + { + label: "100k", + url: "http://localhost:18765/fixtures/synthetic-100k.parquet", + }, + ], + [ + { + label: "1m", + url: "http://localhost:18765/fixtures/synthetic-1m.parquet", + }, + ], + [ + { + label: "10m", + url: "http://localhost:18765/fixtures/synthetic-10m.parquet", + }, + ], + [ + { + label: "10m", + url: "http://localhost:18765/fixtures/synthetic-10m.parquet", + }, + { + label: "10m_2", + url: "http://localhost:18765/fixtures/synthetic-10m-copy.parquet", + }, + ], + [ + { + label: "20m", + url: "http://localhost:18765/fixtures/synthetic-20m.parquet", + }, + ], +]; -const missing = SCALES.filter( - (scale) => !fs.existsSync(path.join(FIXTURES_DIR, `synthetic-${scale}.parquet`)) +const inputFiles = new Set( + TEST_CASES.flat().flatMap(({ url }) => url.replace("http://localhost:18765/fixtures/", "")) +); + +const missing = Array.from(inputFiles).filter( + (fileName) => !fs.existsSync(path.join(FIXTURES_DIR, fileName)) ); if (missing.length > 0) { console.error( - `Missing fixture files: ${missing.map((s) => `synthetic-${s}.parquet`).join(", ")}\n` + + `Missing fixture files: ${missing.join(", ")}\n` + `Download them to ${FIXTURES_DIR} before running this script.` ); process.exit(1); } -const sources = SCALES.map((scale) => ({ label: scale, url: LOCAL_FIXTURE_MAP[scale] })); - -function getCurrentBranch() { +function getCurrentBranch(): string { if (process.env.BENCHMARK_BRANCH) return process.env.BENCHMARK_BRANCH; try { return execSync("git rev-parse --abbrev-ref HEAD", { stdio: "pipe" }).toString().trim(); @@ -39,11 +71,11 @@ function getCurrentBranch() { } } -function slugify(branch) { +function slugify(branch: string): string { return branch.replace(/[^a-zA-Z0-9._-]/g, "-").replace(/-+/g, "-"); } -function getArgValue(flag) { +function getArgValue(flag: string): number | undefined { const idx = process.argv.indexOf(flag); return idx !== -1 ? parseInt(process.argv[idx + 1], 10) : undefined; } @@ -58,7 +90,7 @@ async function main() { if (warmup !== undefined) console.log(`[regression] Warmup rounds: ${warmup}`); const rawResults = await runBenchmarkPage({ - sources, + testCases: TEST_CASES, skipBuild, iterations, warmupRounds: warmup, @@ -77,7 +109,4 @@ async function main() { console.log(`[regression] Results written to ${path.relative(process.cwd(), outFile)}`); } -main().catch((err) => { - console.error("[fatal]", err.message); - process.exit(1); -}); +main(); diff --git a/packages/web/scripts/summarize-results.js b/packages/web/scripts/summarize-results.js index 4bd55935a..5b0e83d89 100644 --- a/packages/web/scripts/summarize-results.js +++ b/packages/web/scripts/summarize-results.js @@ -13,20 +13,12 @@ function defaultResultsFile() { const local = path.join(__dirname, "..", "benchmark-results-local.json"); if (fs.existsSync(local)) return local; - try { - const branch = - process.env.BENCHMARK_BRANCH || - execSync("git rev-parse --abbrev-ref HEAD", { stdio: "pipe" }).toString().trim(); - const slug = branch.replace(/[^a-zA-Z0-9._-]/g, "-").replace(/-+/g, "-"); - const stamped = path.join(__dirname, "..", `benchmark-results-${slug}.json`); - if (fs.existsSync(stamped)) return stamped; - } catch (e) { - if (e instanceof SomeSpecificError) { - return path.join(__dirname, "..", "benchmark-results.json"); - } else { - throw e; - } - } + const branch = + process.env.BENCHMARK_BRANCH || + execSync("git rev-parse --abbrev-ref HEAD", { stdio: "pipe" }).toString().trim(); + const slug = branch.replace(/[^a-zA-Z0-9._-]/g, "-").replace(/-+/g, "-"); + const stamped = path.join(__dirname, "..", `benchmark-results-${slug}.json`); + if (fs.existsSync(stamped)) return stamped; } const file = defaultResultsFile(); @@ -62,24 +54,28 @@ console.log(`Timestamp: ${data.timestamp}`); console.log(`DuckDB init: ${fmt(data.initTimeMs)}`); console.log(""); -const sourceLabels = data.sources.map((s) => s.label); +const sourceLabels = data.results.map((s) => s.labels); // Header row const COL_W = 20; -console.log(col(" Query", 26) + sourceLabels.map((l) => rcol(l, COL_W)).join("")); +console.log( + col(" Query", 26) + sourceLabels.map((labels) => rcol(labels.join(", "), COL_W)).join("") +); console.log(" " + "─".repeat(24 + sourceLabels.length * COL_W)); // Registration row -const regCells = data.sources.map((s) => rcol(fmt(s.registrationMs), COL_W)); +const regCells = data.results.map((result) => rcol(fmt(result.registrationMs), COL_W)); console.log(" " + col("registration", 24) + regCells.join("")); console.log(""); // Query rows (p50 / p95) -const queryNames = [...new Set(data.sources.flatMap((s) => s.queries.map((q) => q.name)))]; +const queryNames = [ + ...new Set(data.results.flatMap((result) => result.queries.map((q) => q.name))), +]; for (const name of queryNames) { - const cells = data.sources.map((s) => { - const q = s.queries.find((x) => x.name === name); + const cells = data.results.map((result) => { + const q = result.queries.find((x) => x.name === name); if (!q) return rcol("—", COL_W); return rcol(`${fmt(q.p50)} / ${fmt(q.p95)}`, COL_W); }); @@ -89,8 +85,8 @@ for (const name of queryNames) { console.log(""); console.log(SEP); console.log( - ` ${data.sources.length} source(s) · ${queryNames.length} queries · ` + - `${data.sources[0]?.queries[0]?.timings?.length ?? "?"} iterations each` + ` ${data.results.length} test case(s) · ${queryNames.length} queries · ` + + `${data.results[0]?.queries[0]?.timings?.length ?? "?"} iterations each` ); console.log(" Timings shown as p50 / p95"); console.log("");