diff --git a/package-lock.json b/package-lock.json index 11f3bae6..0188eb47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "name": "dbx-js-tools", "workspaces": [ "packages/bson-bench", - "packages/eslint-config" + "packages/driver-bench" ], "devDependencies": { "@tsconfig/node16": "^16.1.3", @@ -793,6 +793,14 @@ "dev": true, "license": "ISC" }, + "node_modules/bson": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.7.0.tgz", + "integrity": "sha512-w2IquM5mYzYZv6rs3uN2DZTOBe2a0zXLj53TGDqwF4l6Sz/XsISrisXOJihArF9+BZ6Cq/GjVht7Sjfmri7ytQ==", + "engines": { + "node": ">=16.20.1" + } + }, "node_modules/bson-bench": { "resolved": "packages/bson-bench", "link": true @@ -1086,6 +1094,10 @@ "node": ">=6.0.0" } }, + "node_modules/driver-bench": { + "resolved": "packages/driver-bench", + "link": true + }, "node_modules/emoji-regex": { "version": "8.0.0", "dev": true, @@ -3635,6 +3647,16 @@ "node": ">=6.9.0" } }, + "packages/driver-bench": { + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "bson": "^6.7.0" + }, + "bin": { + "mndb": "lib/cli.mjs" + } + }, "packages/eslint-config": { "version": "1.0.0", "extraneous": true, diff --git a/packages/driver-bench/.gitignore b/packages/driver-bench/.gitignore index 88edb628..89106981 100644 --- a/packages/driver-bench/.gitignore +++ b/packages/driver-bench/.gitignore @@ -1,2 +1,3 @@ node_modules/ lib/ +results.json diff --git a/packages/driver-bench/src/cli.mts b/packages/driver-bench/src/cli.mts index 9f0bf44e..f7fbdbe0 100644 --- a/packages/driver-bench/src/cli.mts +++ b/packages/driver-bench/src/cli.mts @@ -1,24 +1,54 @@ #!/usr/bin/env node -import child_process from 'node:child_process'; -import events from 'node:events'; -import module from 'node:module'; -import path from 'node:path'; +import os from 'node:os'; import process from 'node:process'; +import util from 'node:util'; -let require; +import { main } from './driverBench/index.mjs'; -let { MONGODB_DRIVER_PATH = '' } = process.env; -const { DEBUG_BENCH = 'no' } = process.env; +const argv = util.parseArgs({ + args: process.argv.slice(2), + options: { + grep: { type: 'string', short: 'g', default: '' }, + help: { type: 'boolean', short: 'h', default: false } + }, + strict: true, + allowPositionals: true +}); + +const [driverPath] = argv.positionals; -if (MONGODB_DRIVER_PATH === '') { - require ??= module.createRequire(import.meta.dirname); - MONGODB_DRIVER_PATH = require.resolve('mongodb'); +if (argv.values.help || driverPath == null || driverPath.length === 0) { + console.error('node cli.mjs DRIVER_PATH [--grep/-g BENCHMARK_NAME]'); + process.exit(0); } -const benchmark = child_process.fork(path.join(import.meta.dirname, './driverBench/index.mjs'), { - execArgv: DEBUG_BENCH === 'yes' ? ['--enable-source-maps'] : [], - stdio: 'inherit', - env: { MONGODB_DRIVER_PATH } -}); -await events.once(benchmark, 'exit'); +const hw = os.cpus(); +const ram = os.totalmem() / 1024 ** 3; +const platform = { name: hw[0].model, cores: hw.length, ram: `${ram}GB` }; + +const systemInfo = () => + [ + `\n- cpu: ${platform.name}`, + `- cores: ${platform.cores}`, + `- arch: ${os.arch()}`, + `- os: ${process.platform} (${os.release()})`, + `- ram: ${platform.ram}\n` + ].join('\n'); + +const mongodbDriver = await import(driverPath); + +let { MONGODB_URI = '' } = process.env; +MONGODB_URI = MONGODB_URI.length === 0 ? 'mongodb://127.0.0.1:27017' : MONGODB_URI; + +const client = new mongodbDriver.MongoClient(MONGODB_URI, { serverSelectionTimeoutMS: 2000 }); +try { + console.log('server version:', (await client.db('admin').command({ buildInfo: 1 })).version); + await client.close(); + console.log(`Benching driver at: ${driverPath}`); + console.log(systemInfo()); + await main(argv, mongodbDriver); +} catch (error) { + await client.close(); + console.log('Unable to benchmark against:\n', MONGODB_URI, '\n', error); +} diff --git a/packages/driver-bench/src/driverBench/common.mts b/packages/driver-bench/src/driverBench/common.mts index 649918fb..ef6fcb46 100644 --- a/packages/driver-bench/src/driverBench/common.mts +++ b/packages/driver-bench/src/driverBench/common.mts @@ -4,10 +4,8 @@ import process from 'node:process'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -const { MONGODB_DRIVER_PATH = '' } = process.env; -const { MongoClient, GridFSBucket } = await import( - MONGODB_DRIVER_PATH.length !== 0 ? MONGODB_DRIVER_PATH : 'mongodb' -); +let { MONGODB_URI = '' } = process.env; +MONGODB_URI = MONGODB_URI.length === 0 ? 'mongodb://127.0.0.1:27017' : MONGODB_URI; const DB_NAME = 'perftest'; const COLLECTION_NAME = 'corpus'; @@ -26,10 +24,10 @@ export function loadSpecString(filePath) { return loadSpecFile(filePath, 'utf8'); } -export function makeClient() { - this.client = new MongoClient(process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017', { - serverSelectionTimeoutMS: 2000 - }); +export function makeMakeClient({ MongoClient }) { + return function () { + this.client = new MongoClient(MONGODB_URI, { serverSelectionTimeoutMS: 2000 }); + }; } export function connectClient() { @@ -64,8 +62,10 @@ export function dropCollection() { }); } -export function initBucket() { - this.bucket = new GridFSBucket(this.db); +export function makeInitBucket({ GridFSBucket }) { + return function () { + this.bucket = new GridFSBucket(this.db); + }; } export function dropBucket() { diff --git a/packages/driver-bench/src/driverBench/index.mts b/packages/driver-bench/src/driverBench/index.mts index 891c79e2..a62a6622 100644 --- a/packages/driver-bench/src/driverBench/index.mts +++ b/packages/driver-bench/src/driverBench/index.mts @@ -1,6 +1,5 @@ import { writeFile } from 'node:fs/promises'; -import os from 'node:os'; -import { inspect } from 'node:util'; +import util from 'node:util'; import { Runner } from '../mongoBench/index.mjs'; import { @@ -9,93 +8,74 @@ import { makeSingleBench } from '../mongoBench/suites/index.mjs'; -const bsonType = 'js-bson'; -// TODO(NODE-4606): test against different driver configurations in CI -const hw = os.cpus(); -const ram = os.totalmem() / 1024 ** 3; -const platform = { name: hw[0].model, cores: hw.length, ram: `${ram}GB` }; - -const systemInfo = () => - [ - `\n- cpu: ${platform.name}`, - `- cores: ${platform.cores}`, - `- arch: ${os.arch()}`, - `- os: ${process.platform} (${os.release()})`, - `- ram: ${platform.ram}\n` - ].join('\n'); -console.log(systemInfo()); - -function average(arr) { +function average(arr: number[]) { return arr.reduce((x, y) => x + y, 0) / arr.length; } -const benchmarkRunner = new Runner() - .suite('singleBench', suite => makeSingleBench(suite)) - .suite('multiBench', suite => makeMultiBench(suite)) - .suite('parallel', suite => makeParallelBenchmarks(suite)); +// TODO(NODE-4606): test against different driver configurations in CI +export async function main( + argv: { values: { grep: string } }, + mongodbDriver: { MongoClient; GridFSBucket } +) { + const benchmarkRunner = new Runner(argv.values) + .suite('singleBench', suite => makeSingleBench(suite, mongodbDriver)) + .suite('multiBench', suite => makeMultiBench(suite, mongodbDriver)) + .suite('parallel', suite => makeParallelBenchmarks(suite, mongodbDriver)); -benchmarkRunner - .run() - .then(microBench => { - const singleBench = average([ - microBench.singleBench.findOne, - microBench.singleBench.smallDocInsertOne, - microBench.singleBench.largeDocInsertOne - ]); - const multiBench = average(Object.values(microBench.multiBench)); + const microBench = await benchmarkRunner.run(); - const parallelBench = average([ - microBench.parallel.ldjsonMultiFileUpload, - microBench.parallel.ldjsonMultiFileExport, - microBench.parallel.gridfsMultiFileUpload, - microBench.parallel.gridfsMultiFileDownload - ]); + const singleBench = average([ + microBench.singleBench.findOne, + microBench.singleBench.smallDocInsertOne, + microBench.singleBench.largeDocInsertOne + ]); + const multiBench = average(Object.values(microBench.multiBench)); - const readBench = average([ - microBench.singleBench.findOne, - microBench.multiBench.findManyAndEmptyCursor, - microBench.multiBench.gridFsDownload, - microBench.parallel.gridfsMultiFileDownload, - microBench.parallel.ldjsonMultiFileExport - ]); - const writeBench = average([ - microBench.singleBench.smallDocInsertOne, - microBench.singleBench.largeDocInsertOne, - microBench.multiBench.smallDocBulkInsert, - microBench.multiBench.largeDocBulkInsert, - microBench.multiBench.gridFsUpload, - microBench.parallel.ldjsonMultiFileUpload, - microBench.parallel.gridfsMultiFileUpload - ]); + const parallelBench = average([ + microBench.parallel.ldjsonMultiFileUpload, + microBench.parallel.ldjsonMultiFileExport, + microBench.parallel.gridfsMultiFileUpload, + microBench.parallel.gridfsMultiFileDownload + ]); - const driverBench = average([readBench, writeBench]); + const readBench = average([ + microBench.singleBench.findOne, + microBench.multiBench.findManyAndEmptyCursor, + microBench.multiBench.gridFsDownload, + microBench.parallel.gridfsMultiFileDownload, + microBench.parallel.ldjsonMultiFileExport + ]); + const writeBench = average([ + microBench.singleBench.smallDocInsertOne, + microBench.singleBench.largeDocInsertOne, + microBench.multiBench.smallDocBulkInsert, + microBench.multiBench.largeDocBulkInsert, + microBench.multiBench.gridFsUpload, + microBench.parallel.ldjsonMultiFileUpload, + microBench.parallel.gridfsMultiFileUpload + ]); - const benchmarkResults = { - singleBench, - multiBench, - parallelBench, - readBench, - writeBench, - driverBench, - ...microBench.parallel, - ...microBench.bsonBench, - ...microBench.singleBench, - ...microBench.multiBench - }; + const driverBench = average([readBench, writeBench]); - return Object.entries(benchmarkResults).map(([benchmarkName, result]) => { - return { - info: { - test_name: benchmarkName, - tags: [bsonType] - }, - metrics: [{ name: 'megabytes_per_second', value: result }] - }; - }); - }) - .then(data => { - const results = JSON.stringify(data, undefined, 2); - console.log(inspect(data, { depth: Infinity, colors: true })); - return writeFile('results.json', results); - }) - .catch(err => console.error(err)); + const benchmarkResults = { + singleBench, + multiBench, + parallelBench, + readBench, + writeBench, + driverBench, + ...microBench.parallel, + ...microBench.bsonBench, + ...microBench.singleBench, + ...microBench.multiBench + }; + + const data = Object.entries(benchmarkResults).map(([benchmarkName, result]) => ({ + info: { test_name: benchmarkName, tags: ['js-bson'] }, + metrics: [{ name: 'megabytes_per_second', value: result }] + })); + + const results = JSON.stringify(data, undefined, 2); + console.log(util.inspect(data, { depth: Infinity, colors: true })); + await writeFile('results.json', results); +} diff --git a/packages/driver-bench/src/mod.mts b/packages/driver-bench/src/mod.mts index e2f8967b..f2f85839 100644 --- a/packages/driver-bench/src/mod.mts +++ b/packages/driver-bench/src/mod.mts @@ -1,4 +1,4 @@ -// TODO: This module is not really meant to be imported the CLI is currently the focus of how to utilize this code +// TODO: This module is not really meant to be imported. The CLI is currently the focus of how to utilize this code // See wishlist in readme export { Benchmark } from './mongoBench/benchmark.mjs'; export { Runner } from './mongoBench/runner.mjs'; diff --git a/packages/driver-bench/src/mongoBench/runner.mts b/packages/driver-bench/src/mongoBench/runner.mts index b0ff2fff..1daded74 100644 --- a/packages/driver-bench/src/mongoBench/runner.mts +++ b/packages/driver-bench/src/mongoBench/runner.mts @@ -53,6 +53,8 @@ export type RunnerOptions = { maxExecutionTime?: number; minExecutionCount?: number; reporter?: (message?: any, ...optionalParams: any[]) => void; + /** Only run TEST NAMES that match */ + grep?: string; }; export class Runner { @@ -61,13 +63,14 @@ export class Runner { private minExecutionCount: number; private reporter: (message?: any, ...optionalParams: any[]) => void; private children: Record; + private grep: RegExp | null; - constructor(options?: RunnerOptions) { - options = options || {}; + constructor(options: RunnerOptions = {}) { this.minExecutionTime = options.minExecutionTime || CONSTANTS.DEFAULT_MIN_EXECUTION_TIME; this.maxExecutionTime = options.maxExecutionTime || CONSTANTS.DEFAULT_MAX_EXECUTION_TIME; this.minExecutionCount = options.minExecutionCount || CONSTANTS.DEFAULT_MIN_EXECUTION_COUNT; this.reporter = options.reporter ?? console.log.bind(console); + this.grep = options.grep ? new RegExp(options.grep, 'i') : null; this.children = {}; } @@ -121,6 +124,11 @@ export class Runner { const result: Record = {}; for (const [name, benchmark] of benchmarks) { + if (this.grep && !this.grep.test(name)) { + this.reporter(` Skipping Benchmark "${name}"`); + result[name] = 0; + continue; + } this.reporter(` Executing Benchmark "${name}"`); result[name] = await this._runBenchmark(benchmark); this.reporter(` Result ${result[name].toFixed(4)} MB/s`); diff --git a/packages/driver-bench/src/mongoBench/suites/multiBench.mts b/packages/driver-bench/src/mongoBench/suites/multiBench.mts index 242d6b65..88d92d6f 100644 --- a/packages/driver-bench/src/mongoBench/suites/multiBench.mts +++ b/packages/driver-bench/src/mongoBench/suites/multiBench.mts @@ -8,14 +8,14 @@ import { dropBucket, dropCollection, dropDb, - initBucket, initCollection, initDb, loadSpecFile, - makeClient, + makeInitBucket, makeLoadInsertDocs, makeLoadJSON, makeLoadTweets, + makeMakeClient, writeSingleByteFileToBucket } from '../../driverBench/common.mjs'; import type { Suite } from '../suite.mjs'; @@ -34,19 +34,19 @@ async function gridFsUpload() { await pipeline(uploadData, uploadStream); } -export function makeMultiBench(suite: Suite): Suite { +export function makeMultiBench(suite: Suite, mongodbDriver): Suite { return suite .benchmark('findManyAndEmptyCursor', benchmark => benchmark .taskSize(16.22) .setup(makeLoadJSON('tweet.json')) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb) .setup(initCollection) .setup(makeLoadTweets(false)) - .task(async function () { + .task(async function findManyAndEmptyCursor() { // eslint-disable-next-line @typescript-eslint/no-unused-vars for await (const _ of this.collection.find({})) { // do nothing @@ -60,7 +60,7 @@ export function makeMultiBench(suite: Suite): Suite { .taskSize(2.75) .setup(makeLoadJSON('small_doc.json')) .setup(makeLoadInsertDocs(10000)) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb) @@ -70,7 +70,7 @@ export function makeMultiBench(suite: Suite): Suite { .beforeTask(dropCollection) .beforeTask(createCollection) .beforeTask(initCollection) - .task(async function () { + .task(async function smallDocBulkInsert() { await this.collection.insertMany(this.docs, { ordered: true }); }) .teardown(dropDb) @@ -81,7 +81,7 @@ export function makeMultiBench(suite: Suite): Suite { .taskSize(27.31) .setup(makeLoadJSON('large_doc.json')) .setup(makeLoadInsertDocs(10)) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb) @@ -91,7 +91,7 @@ export function makeMultiBench(suite: Suite): Suite { .beforeTask(dropCollection) .beforeTask(createCollection) .beforeTask(initCollection) - .task(async function () { + .task(async function largeDocBulkInsert() { await this.collection.insertMany(this.docs, { ordered: true }); }) .teardown(dropDb) @@ -101,14 +101,14 @@ export function makeMultiBench(suite: Suite): Suite { benchmark .taskSize(52.43) .setup(loadGridFs) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb) .setup(initDb) .setup(initCollection) .beforeTask(dropBucket) - .beforeTask(initBucket) + .beforeTask(makeInitBucket(mongodbDriver)) .beforeTask(gridFsInitUploadStream) .beforeTask(writeSingleByteFileToBucket) .task(gridFsUpload) @@ -119,21 +119,21 @@ export function makeMultiBench(suite: Suite): Suite { benchmark .taskSize(52.43) .setup(loadGridFs) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb) .setup(initDb) .setup(initCollection) .setup(dropBucket) - .setup(initBucket) + .setup(makeInitBucket(mongodbDriver)) .setup(gridFsInitUploadStream) .setup(async function () { await gridFsUpload.call(this); this.id = this.uploadStream.id; this.uploadData = undefined; }) - .task(async function () { + .task(async function gridFsDownload() { // eslint-disable-next-line @typescript-eslint/no-unused-vars for await (const _ of this.bucket.openDownloadStream(this.id)) { // do nothing @@ -141,5 +141,21 @@ export function makeMultiBench(suite: Suite): Suite { }) .teardown(dropDb) .teardown(disconnectClient) + ) + .benchmark('findManyAndToArray', benchmark => + benchmark + .taskSize(16.22) + .setup(makeLoadJSON('tweet.json')) + .setup(makeMakeClient(mongodbDriver)) + .setup(connectClient) + .setup(initDb) + .setup(dropDb) + .setup(initCollection) + .setup(makeLoadTweets(false)) + .task(async function () { + await this.collection.find({}).toArray(); + }) + .teardown(dropDb) + .teardown(disconnectClient) ); } diff --git a/packages/driver-bench/src/mongoBench/suites/parallelBench.mts b/packages/driver-bench/src/mongoBench/suites/parallelBench.mts index e2e84b1a..f443d80e 100644 --- a/packages/driver-bench/src/mongoBench/suites/parallelBench.mts +++ b/packages/driver-bench/src/mongoBench/suites/parallelBench.mts @@ -14,10 +14,10 @@ import { dropBucket, dropCollection, dropDb, - initBucket, initCollection, initDb, - makeClient + makeInitBucket, + makeMakeClient } from '../../driverBench/common.mjs'; import type { Suite } from '../suite.mjs'; @@ -117,13 +117,13 @@ async function gridfsMultiFileDownload() { await Promise.all(downloads); } -export function makeParallelBenchmarks(suite: Suite): Suite { +export function makeParallelBenchmarks(suite: Suite, mongodbDriver): Suite { return suite .benchmark('ldjsonMultiFileUpload', benchmark => // https://github.com/mongodb/specifications/blob/master/source/benchmarking/benchmarking.rst#ldjson-multi-file-import benchmark .taskSize(565) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb) @@ -138,7 +138,7 @@ export function makeParallelBenchmarks(suite: Suite): Suite { // https://github.com/mongodb/specifications/blob/master/source/benchmarking/benchmarking.rst#ldjson-multi-file-export benchmark .taskSize(565) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb) @@ -159,14 +159,14 @@ export function makeParallelBenchmarks(suite: Suite): Suite { // https://github.com/mongodb/specifications/blob/master/source/benchmarking/benchmarking.rst#gridfs-multi-file-upload benchmark .taskSize(262.144) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb) .setup(initDb) .setup(initCollection) .beforeTask(dropBucket) - .beforeTask(initBucket) + .beforeTask(makeInitBucket(mongodbDriver)) .beforeTask(async function () { const stream = this.bucket.openUploadStream('setup-file.txt'); const oneByteFile = Readable.from('a'); @@ -180,7 +180,7 @@ export function makeParallelBenchmarks(suite: Suite): Suite { // https://github.com/mongodb/specifications/blob/master/source/benchmarking/benchmarking.rst#gridfs-multi-file-download benchmark .taskSize(262.144) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb) @@ -188,10 +188,10 @@ export function makeParallelBenchmarks(suite: Suite): Suite { .setup(initCollection) .setup(initTemporaryDirectory) .setup(dropBucket) - .setup(initBucket) + .setup(makeInitBucket(mongodbDriver)) .setup(gridfsMultiFileUpload) .beforeTask(clearTemporaryDirectory) - .setup(initBucket) + .setup(makeInitBucket(mongodbDriver)) .task(gridfsMultiFileDownload) .teardown(dropDb) .teardown(async function () { diff --git a/packages/driver-bench/src/mongoBench/suites/singleBench.mts b/packages/driver-bench/src/mongoBench/suites/singleBench.mts index ae7612b7..9714bff9 100644 --- a/packages/driver-bench/src/mongoBench/suites/singleBench.mts +++ b/packages/driver-bench/src/mongoBench/suites/singleBench.mts @@ -6,18 +6,18 @@ import { dropDb, initCollection, initDb, - makeClient, makeLoadJSON, - makeLoadTweets + makeLoadTweets, + makeMakeClient } from '../../driverBench/common.mjs'; import type { Suite } from '../suite.mjs'; -export function makeSingleBench(suite: Suite): Suite { +export function makeSingleBench(suite: Suite, mongodbDriver): Suite { return suite .benchmark('runCommand', benchmark => benchmark .taskSize(0.16) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .task(async function () { @@ -31,7 +31,7 @@ export function makeSingleBench(suite: Suite): Suite { benchmark .taskSize(16.22) .setup(makeLoadJSON('tweet.json')) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb) @@ -49,7 +49,7 @@ export function makeSingleBench(suite: Suite): Suite { benchmark .taskSize(2.75) .setup(makeLoadJSON('small_doc.json')) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb) @@ -74,7 +74,7 @@ export function makeSingleBench(suite: Suite): Suite { benchmark .taskSize(27.31) .setup(makeLoadJSON('large_doc.json')) - .setup(makeClient) + .setup(makeMakeClient(mongodbDriver)) .setup(connectClient) .setup(initDb) .setup(dropDb)