diff --git a/meerkat-node/src/__tests__/benchmarking/benchmarking.parquet b/meerkat-node/src/__tests__/benchmarking/benchmarking.parquet new file mode 100644 index 00000000..ec190aa2 Binary files /dev/null and b/meerkat-node/src/__tests__/benchmarking/benchmarking.parquet differ diff --git a/meerkat-node/src/__tests__/benchmarking/benchmarking.spec.ts b/meerkat-node/src/__tests__/benchmarking/benchmarking.spec.ts new file mode 100644 index 00000000..db164944 --- /dev/null +++ b/meerkat-node/src/__tests__/benchmarking/benchmarking.spec.ts @@ -0,0 +1,119 @@ +import { duckdbExec } from "../../duckdb-exec"; +const { + PARQUET_FILE_PATH, + getOldQueries, + getNewQueries +} = require('./constants') + +const TABLE_NAME = 'benchmarking' + +const RUNS_PER_QUERY = 20 +const EXECUTION_INDEX = 5; + +type RunInfo = { + end: number; + start: number; +} + +const getParquetSQL = (parquetPath: string, TABLE_NAME: string) => `CREATE TABLE ${TABLE_NAME} AS SELECT * FROM read_parquet('${parquetPath}');` as const + +const getAverageDuration = (results: { end: number, start: number }[]) => { + let totalDuration = 0; + results.forEach((result) => { + const duration = result.end - result.start; + totalDuration += duration; + }) + const avgDuration = totalDuration / results.length; + return avgDuration; +} + +const getAverageImprovement = (oldRuns: RunInfo[], newRuns: RunInfo[]) => { + const oldAvgDuration = getAverageDuration(oldRuns) + const newAvgDuration = getAverageDuration(newRuns) + // Calculate improvement + const improvement = ((oldAvgDuration - newAvgDuration) / oldAvgDuration) * 100; + return { + oldAvgDuration, + newAvgDuration, + improvement + } +} + + +const getPercentileItem = (runs: RunInfo[], percentile: number) => { + const sortedRuns = runs.sort((a, b) => a.end - a.start - (b.end - b.start)) + const p90Index = Math.floor(runs.length * percentile) + return sortedRuns[p90Index] +} + +const getPercentileImprovement = (oldRuns: RunInfo[], newRuns: RunInfo[], percentile: number) => { + const { end: oldEndP90, start: oldStartP90 } = getPercentileItem(oldRuns, percentile) + const { end: newEndP90, start: newStartP90 } = getPercentileItem(newRuns, percentile) + const oldDuration = oldEndP90 - oldStartP90 + const newDuration = newEndP90 - newStartP90 + const improvement = ((oldDuration - newDuration) / oldDuration) * 100; + return { + oldDuration, + newDuration, + improvement + } +} + +const runQueryWithTiming = async (sql: string, runs: number): Promise => { + const runsArray = new Array(runs).fill(undefined) + const dataPromises = runsArray.map(async () => { + const start = performance.now(); + await duckdbExec(sql); + const end = performance.now(); + return { + end, + start + } + }) + const results = await Promise.all(dataPromises); + return results + +} + + +const benchmarkQueries = async (tableName: string, runsPerQuery: number) => { + const oldQueries = getOldQueries(tableName); + const newQueries = getNewQueries(tableName); + + const oldQuery = oldQueries[EXECUTION_INDEX]; + const newQuery = newQueries[EXECUTION_INDEX]; + + console.log({ oldQuery, newQuery }) + // Run old query multiple times and get average + const oldRuns = await runQueryWithTiming(oldQuery.sql, runsPerQuery); + + // Run new query multiple times and get average + const newRuns = await runQueryWithTiming(newQuery.sql, runsPerQuery); + + const avgImprovement = getAverageImprovement(oldRuns, newRuns) + const p50Improvement = getPercentileImprovement(oldRuns, newRuns, 0.5) + const p90Improvement = getPercentileImprovement(oldRuns, newRuns, 0.9) + + + return { + avg: avgImprovement, + p50: p50Improvement, + p90: p90Improvement + } + +} + + + +describe('benchmarking', () => { + beforeAll(async () => { + const sql = getParquetSQL(PARQUET_FILE_PATH, TABLE_NAME); + await duckdbExec(sql); + }) + + it('should be able to run benchmarks', async () => { + const result = await benchmarkQueries(TABLE_NAME, RUNS_PER_QUERY) + console.log(result) + }) +}) + diff --git a/meerkat-node/src/__tests__/benchmarking/constants.js b/meerkat-node/src/__tests__/benchmarking/constants.js new file mode 100644 index 00000000..795f03b8 --- /dev/null +++ b/meerkat-node/src/__tests__/benchmarking/constants.js @@ -0,0 +1,182 @@ + + + +const getOldQueries = ( + tableName +) => { + return [ + { + name: "Simple COUNT", + sql: ` + SELECT COUNT(*) FROM ( + SELECT * FROM ( + SELECT * FROM ${tableName} + ) + )` + }, + { + name: "Aggregation with GROUP BY", + sql: ` + SELECT ${tableName}__customer_id, SUM(total_amount) AS total_spent FROM ( + SELECT *, customer_id as ${tableName}__customer_id FROM ( + SELECT * FROM ${tableName} + ) + ) GROUP BY ${tableName}__customer_id ORDER BY total_spent DESC` + }, + { + name: "Multiple Aggregations with GROUP BY", + sql: `SELECT product_id, product_name, SUM(quantity) AS total_quantity, AVG(product_price) AS avg_price FROM ( + SELECT *, product_id AS ${tableName}__product_id FROM ( + SELECT * FROM ${tableName} + ) + ) GROUP BY product_id, product_name ORDER BY total_quantity DESC` + }, + { + name: "Date Truncation and Aggregation", + sql: ` + SELECT ${tableName}__month, SUM(total_amount) AS monthly_revenue FROM ( + SELECT *, DATE_TRUNC('month', order_date) AS ${tableName}__month FROM ( + SELECT * FROM ${tableName} + ) + ) GROUP BY ${tableName}__month ORDER BY ${tableName}__month` + }, + { + name: "Window Function", + sql: ` + SELECT + ${tableName}__order_id, + ${tableName}__order_date, + ${tableName}__total_amount, + SUM(total_amount) OVER (ORDER BY order_date) AS ${tableName}__cumulative_revenue + FROM ( + SELECT *, order_id AS ${tableName}__order_id, order_date AS ${tableName}__order_date, total_amount AS ${tableName}__total_amount FROM ( + SELECT * FROM ${tableName} + ) + ) + ORDER BY ${tableName}__order_date + LIMIT 100 + ` + }, { + name: 'filters', + sql: ` + SELECT + DATE_TRUNC('month', ${tableName}.order_date) AS ${tableName}__order_month, + ${tableName}__customer_id, + ${tableName}__product_id, + ${tableName}__product_name, + COUNT(DISTINCT ${tableName}.order_id) AS ${tableName}__total_orders, + SUM(${tableName}.quantity) AS ${tableName}__quantity, + AVG(${tableName}.product_price) AS ${tableName}__avg_product_price, + FROM ( + select + *, + ${tableName}.customer_id AS ${tableName}__customer_id, + ${tableName}.product_id AS ${tableName}__product_id, + ${tableName}.product_name AS ${tableName}__product_name, + ${tableName}.order_date AS ${tableName}__order_date, + from ( + select * from ${tableName} + ) AS ${tableName} + ) AS ${tableName} + WHERE + ${tableName}__order_date BETWEEN '2023-01-01 00:00:00' AND '2023-06-30 23:59:59' + AND ${tableName}__product_id BETWEEN 100 AND 200 + AND ${tableName}__product_name LIKE '%Premium%' + GROUP BY + ${tableName}__order_month, + ${tableName}__customer_id, + ${tableName}__product_id, + ${tableName}__product_name + HAVING + ${tableName}__total_orders > 10 + ORDER BY + ${tableName}__total_orders DESC + ` + } + + ]; +} + +const getNewQueries = (tableName: string) => { + return [ + { + name: "Simple COUNT", + sql: `SELECT COUNT(*) FROM ${tableName}` + }, + { + name: "Aggregation with GROUP BY", + sql: `SELECT customer_id, SUM(total_amount) AS total_spent FROM ${tableName} GROUP BY customer_id ORDER BY total_spent DESC` + }, + { + name: "Multiple Aggregations with GROUP BY", + sql: `SELECT product_id, product_name, SUM(quantity) AS total_quantity, AVG(product_price) AS avg_price FROM ${tableName} GROUP BY product_id, product_name ORDER BY total_quantity DESC` + }, + { + name: "Date Truncation and Aggregation", + sql: `SELECT DATE_TRUNC('month', order_date) AS month, SUM(total_amount) AS monthly_revenue FROM ${tableName} GROUP BY month ORDER BY month` + }, + { + name: "Window Function", + sql: ` + SELECT + order_id, + order_date, + total_amount, + SUM(total_amount) OVER (ORDER BY order_date) AS cumulative_revenue + FROM ${tableName} + ORDER BY order_date + LIMIT 100 + ` + }, + { + name: 'filters', + sql: ` + SELECT + ${tableName}__order_month, + ${tableName}.customer_id, + ${tableName}.product_id, + ${tableName}.product_name, + COUNT(DISTINCT ${tableName}.order_id) AS benchmarking__total_orders, + SUM(${tableName}.quantity) AS benchmarking__quantity, + AVG(${tableName}.product_price) AS benchmarking__avg_product_price + FROM + ( + SELECT + DATE_TRUNC('month', ${tableName}.order_date) AS ${tableName}__order_month, + ${tableName}.order_date, + ${tableName}.customer_id, + ${tableName}.product_id, + ${tableName}.product_name, + ${tableName}.order_id, + ${tableName}.quantity, + ${tableName}.product_price + from ${tableName} + ) AS ${tableName} + WHERE + ${tableName}__order_month BETWEEN '2023-01-01' AND '2023-06-30' + AND ${tableName}.product_id BETWEEN 100 AND 200 + AND ${tableName}.product_name LIKE '%Premium%' + GROUP BY + ${tableName}__order_month, + ${tableName}.customer_id, + ${tableName}.product_id, + ${tableName}.product_name + HAVING + COUNT(DISTINCT ${tableName}.order_id) > 10 + ORDER BY + COUNT(DISTINCT ${tableName}.order_id) DESC` + } + ]; +} + + + +module.exports = { + PARQUET_FILE_PATH: '/Users/zaidjan/Documents/Projects/meerkat/meerkat-node/src/__tests__/benchmarking/benchmarking.parquet', + getOldQueries, + getNewQueries +} + + + + diff --git a/meerkat-node/src/__tests__/benchmarking/generate-data.js b/meerkat-node/src/__tests__/benchmarking/generate-data.js new file mode 100644 index 00000000..90fcb35e --- /dev/null +++ b/meerkat-node/src/__tests__/benchmarking/generate-data.js @@ -0,0 +1,104 @@ +const parquet = require('parquetjs'); +const { PARQUET_FILE_PATH } = require('./constants'); + +function randomDate(start, end) { + return new Date(start.getTime() + Math.random() * (end.getTime() - start.getTime())); +} + +function generateAdditionalColumns(baseSchema, additionalColumnCount) { + const dataTypes = ['INT64', 'DOUBLE', 'BOOLEAN', 'UTF8', 'TIMESTAMP_MILLIS']; + const prefixes = ['extra', 'aux', 'meta', 'sup', 'add']; + + for (let i = 0; i < additionalColumnCount; i++) { + const prefix = prefixes[Math.floor(Math.random() * prefixes.length)]; + const columnName = `${prefix}_${i + 1}`; + const dataType = dataTypes[Math.floor(Math.random() * dataTypes.length)]; + baseSchema[columnName] = { type: dataType }; + } + + return new parquet.ParquetSchema(baseSchema); +} + +function generateRandomValue(dataType) { + switch (dataType) { + case 'INT64': + return Math.floor(Math.random() * 1000000); + case 'DOUBLE': + return parseFloat((Math.random() * 1000).toFixed(2)); + case 'BOOLEAN': + return Math.random() > 0.5; + case 'UTF8': + return `RandomString_${Math.random().toString(36).substring(2, 10)}`; + case 'TIMESTAMP_MILLIS': + return randomDate(new Date(2020, 0, 1), new Date(2023, 11, 31)); + default: + return null; + } +} + +function generateRandomProductName() { + const adjectives = ['Premium', 'Deluxe', 'Basic', 'Super', 'Ultra', 'Eco', 'Smart', 'Pro']; + const nouns = ['Widget', 'Gadget', 'Tool', 'Device', 'Accessory', 'Appliance', 'Gizmo', 'Component']; + return `${adjectives[Math.floor(Math.random() * adjectives.length)]} ${nouns[Math.floor(Math.random() * nouns.length)]}`; +} + +async function generateData(numRows, additionalColumnCount) { + const baseSchema = { + order_id: { type: 'INT64' }, + order_date: { type: 'TIMESTAMP_MILLIS' }, + customer_id: { type: 'INT64' }, + product_id: { type: 'INT64' }, + product_name: { type: 'UTF8' }, + quantity: { type: 'INT64' }, + product_price: { type: 'DOUBLE' }, + total_amount: { type: 'DOUBLE' } + }; + + const schema = generateAdditionalColumns(baseSchema, additionalColumnCount); + + // Create a new Parquet file writer + const writer = await parquet.ParquetWriter.openFile(schema, PARQUET_FILE_PATH); + + const startDate = new Date(2020, 0, 1); + const endDate = new Date(2023, 11, 31); + + for (let i = 0; i < numRows; i++) { + const orderDate = randomDate(startDate, endDate); + const quantity = Math.floor(Math.random() * 20) + 1; + const productPrice = parseFloat((Math.random() * 999 + 1).toFixed(2)); + const totalAmount = quantity * productPrice; + + const row = { + order_id: i + 1, + order_date: orderDate, + customer_id: Math.floor(Math.random() * 1000000) + 1, + product_id: Math.floor(Math.random() * 10000) + 1, + product_name: generateRandomProductName(), + quantity: quantity, + product_price: productPrice, + total_amount: totalAmount + }; + + // Generate values for additional columns + for (const [columnName, columnDef] of Object.entries(schema.fields)) { + if (!(columnName in row)) { + row[columnName] = generateRandomValue(columnDef.type); + } + } + + await writer.appendRow(row); + + if (i % 100000 === 0) { + console.log(`Processed ${i} records`); + } + } + await writer.close(); + console.log(`Parquet file "benchmark_data.parquet" has been generated with ${numRows} rows and ${Object.keys(schema.fields).length} columns.`); + console.log('Schema:', schema.fields); +} + +// Example usage +const numRows = 1000000; // 10 million rows +const additionalColumnCount = 0; // 10 additional columns + +generateData(numRows, additionalColumnCount).catch(console.error); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index b050ac15..255adf9f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,12 +20,16 @@ "duckdb": "^0.10.2", "express": "^4.19.2", "fake-indexeddb": "^5.0.1", + "faker": "6.6.6", + "fs": "0.0.1-security", "install": "^0.13.0", "lodash": "^4.17.21", "lodash.isnil": "^4.0.0", "loglevel": "^1.8.1", "next": "13.5.4", "npm": "^10.8.2", + "parquet": "0.0.0", + "parquetjs": "0.11.2", "puppeteer": "^21.5.2", "react": "18.2.0", "react-dom": "18.2.0", @@ -8378,6 +8382,12 @@ "node": ">=8" } }, + "node_modules/bindings": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", + "integrity": "sha512-u4cBQNepWxYA55FunZSM7wMi55yQaN0otnhhilNoWHq0MfOfJeQx0v0mRRpolGOExPjZcl6FtB0BB8Xkb88F0g==", + "optional": true + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -8506,6 +8516,14 @@ "node": ">=8" } }, + "node_modules/brotli": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz", + "integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==", + "dependencies": { + "base64-js": "^1.1.2" + } + }, "node_modules/browserslist": { "version": "4.21.10", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.10.tgz", @@ -8558,6 +8576,14 @@ "node-int64": "^0.4.0" } }, + "node_modules/bson": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", + "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==", + "engines": { + "node": ">=0.6.19" + } + }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -12039,6 +12065,11 @@ "node": ">=18" } }, + "node_modules/faker": { + "version": "6.6.6", + "resolved": "https://registry.npmjs.org/faker/-/faker-6.6.6.tgz", + "integrity": "sha512-9tCqYEDHI5RYFQigXFwF1hnCwcWCOJl/hmll0lr5D2Ljjb0o4wphb69wikeJDz5qCEzXCoPvG6ss5SDP6IfOdg==" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -12795,6 +12826,11 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -13792,6 +13828,11 @@ "node": ">= 0.10" } }, + "node_modules/int53": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz", + "integrity": "sha512-a5jlKftS7HUOhkUyYD7j2sJ/ZnvWiNlZS1ldR+g1ifQ+/UuZXIE+YTc/lK1qGj/GwAU5F8Z0e1eVq2t1J5Ob2g==" + }, "node_modules/internal-slot": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.5.tgz", @@ -16298,6 +16339,16 @@ "lz-string": "bin/bin.js" } }, + "node_modules/lzo": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/lzo/-/lzo-0.4.11.tgz", + "integrity": "sha512-apQHNoW2Alg72FMqaC/7pn03I7umdgSVFt2KRkCXXils4Z9u3QBh1uOtl2O5WmZIDLd9g6Lu4lIdOLmiSTFVCQ==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "bindings": "~1.2.1" + } + }, "node_modules/magic-string": { "version": "0.30.11", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.11.tgz", @@ -17204,8 +17255,7 @@ "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", - "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", - "dev": true + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==" }, "node_modules/node-machine-id": { "version": "1.1.12", @@ -19783,6 +19833,14 @@ "node": ">= 0.4" } }, + "node_modules/object-stream": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/object-stream/-/object-stream-0.0.1.tgz", + "integrity": "sha512-+NPJnRvX9RDMRY9mOWOo/NDppBjbZhXirNNSu2IBnuNboClC9h1ZGHXgHBLDbJMHsxeJDq922aVmG5xs24a/cA==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/object.assign": { "version": "4.1.4", "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz", @@ -20279,6 +20337,31 @@ "node": ">=6" } }, + "node_modules/parquet": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/parquet/-/parquet-0.0.0.tgz", + "integrity": "sha512-STx+z5NfXSFNB25j12xkBlgDmRyyce+Ub49hlAWfPIrUtYihPO9OmCv940YGqQvQe5tkL9t/U0tt45FBJan9/A==" + }, + "node_modules/parquetjs": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/parquetjs/-/parquetjs-0.11.2.tgz", + "integrity": "sha512-Y6FOc3Oi2AxY4TzJPz7fhICCR8tQNL3p+2xGQoUAMbmlJBR7+JJmMrwuyMjIpDiM7G8Wj/8oqOH4UDUmu4I5ZA==", + "dependencies": { + "brotli": "^1.3.0", + "bson": "^1.0.4", + "int53": "^0.2.4", + "object-stream": "0.0.1", + "snappyjs": "^0.6.0", + "thrift": "^0.11.0", + "varint": "^5.0.0" + }, + "engines": { + "node": ">=7.6" + }, + "optionalDependencies": { + "lzo": "^0.4.0" + } + }, "node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", @@ -21588,6 +21671,16 @@ } ] }, + "node_modules/q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==", + "deprecated": "You or someone you depend on is using Q, the JavaScript Promise library that gave JavaScript developers strong feelings about promises. They can almost certainly migrate to the native JavaScript promise now. Thank you literally everyone for joining me in this bet against the odds. Be excellent to each other.\n\n(For a CapTP with native promises, see @endo/eventual-send and @endo/captp)", + "engines": { + "node": ">=0.6.0", + "teleport": ">=0.2.0" + } + }, "node_modules/qs": { "version": "6.11.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", @@ -23208,6 +23301,11 @@ "tslib": "^2.0.3" } }, + "node_modules/snappyjs": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/snappyjs/-/snappyjs-0.6.1.tgz", + "integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg==" + }, "node_modules/sockjs": { "version": "0.3.24", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz", @@ -24445,6 +24543,19 @@ "real-require": "^0.1.0" } }, + "node_modules/thrift": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/thrift/-/thrift-0.11.0.tgz", + "integrity": "sha512-UpsBhOC45a45TpeHOXE4wwYwL8uD2apbHTbtBvkwtUU4dNwCjC7DpQTjw2Q6eIdfNtw+dKthdwq94uLXTJPfFw==", + "dependencies": { + "node-int64": "^0.4.0", + "q": "^1.5.0", + "ws": ">= 2.2.3" + }, + "engines": { + "node": ">= 4.1.0" + } + }, "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -25357,6 +25468,11 @@ "node": ">= 0.10" } }, + "node_modules/varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==" + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -33054,6 +33170,12 @@ "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", "devOptional": true }, + "bindings": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", + "integrity": "sha512-u4cBQNepWxYA55FunZSM7wMi55yQaN0otnhhilNoWHq0MfOfJeQx0v0mRRpolGOExPjZcl6FtB0BB8Xkb88F0g==", + "optional": true + }, "bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -33167,6 +33289,14 @@ "fill-range": "^7.0.1" } }, + "brotli": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.3.tgz", + "integrity": "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg==", + "requires": { + "base64-js": "^1.1.2" + } + }, "browserslist": { "version": "4.21.10", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.10.tgz", @@ -33196,6 +33326,11 @@ "node-int64": "^0.4.0" } }, + "bson": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", + "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==" + }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -35731,6 +35866,11 @@ "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-5.0.1.tgz", "integrity": "sha512-vxybH29Owtc6khV/Usy47B1g+eKwyhFiX8nwpCC4td320jvwrKQDH6vNtcJZgUzVxmfsSIlHzLKQzT76JMCO7A==" }, + "faker": { + "version": "6.6.6", + "resolved": "https://registry.npmjs.org/faker/-/faker-6.6.6.tgz", + "integrity": "sha512-9tCqYEDHI5RYFQigXFwF1hnCwcWCOJl/hmll0lr5D2Ljjb0o4wphb69wikeJDz5qCEzXCoPvG6ss5SDP6IfOdg==" + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -36290,6 +36430,11 @@ } } }, + "fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" + }, "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -37025,6 +37170,11 @@ "resolved": "https://registry.npmjs.org/install/-/install-0.13.0.tgz", "integrity": "sha512-zDml/jzr2PKU9I8J/xyZBQn8rPCAY//UOYNmR01XwNwyfhEWObo2SWfSl1+0tm1u6PhxLwDnfsT/6jB7OUxqFA==" }, + "int53": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz", + "integrity": "sha512-a5jlKftS7HUOhkUyYD7j2sJ/ZnvWiNlZS1ldR+g1ifQ+/UuZXIE+YTc/lK1qGj/GwAU5F8Z0e1eVq2t1J5Ob2g==" + }, "internal-slot": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.5.tgz", @@ -38933,6 +39083,15 @@ "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", "dev": true }, + "lzo": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/lzo/-/lzo-0.4.11.tgz", + "integrity": "sha512-apQHNoW2Alg72FMqaC/7pn03I7umdgSVFt2KRkCXXils4Z9u3QBh1uOtl2O5WmZIDLd9g6Lu4lIdOLmiSTFVCQ==", + "optional": true, + "requires": { + "bindings": "~1.2.1" + } + }, "magic-string": { "version": "0.30.11", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.11.tgz", @@ -39614,8 +39773,7 @@ "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", - "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", - "dev": true + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==" }, "node-machine-id": { "version": "1.1.12", @@ -41279,6 +41437,11 @@ "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", "dev": true }, + "object-stream": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/object-stream/-/object-stream-0.0.1.tgz", + "integrity": "sha512-+NPJnRvX9RDMRY9mOWOo/NDppBjbZhXirNNSu2IBnuNboClC9h1ZGHXgHBLDbJMHsxeJDq922aVmG5xs24a/cA==" + }, "object.assign": { "version": "4.1.4", "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz", @@ -41627,6 +41790,26 @@ "callsites": "^3.0.0" } }, + "parquet": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/parquet/-/parquet-0.0.0.tgz", + "integrity": "sha512-STx+z5NfXSFNB25j12xkBlgDmRyyce+Ub49hlAWfPIrUtYihPO9OmCv940YGqQvQe5tkL9t/U0tt45FBJan9/A==" + }, + "parquetjs": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/parquetjs/-/parquetjs-0.11.2.tgz", + "integrity": "sha512-Y6FOc3Oi2AxY4TzJPz7fhICCR8tQNL3p+2xGQoUAMbmlJBR7+JJmMrwuyMjIpDiM7G8Wj/8oqOH4UDUmu4I5ZA==", + "requires": { + "brotli": "^1.3.0", + "bson": "^1.0.4", + "int53": "^0.2.4", + "lzo": "^0.4.0", + "object-stream": "0.0.1", + "snappyjs": "^0.6.0", + "thrift": "^0.11.0", + "varint": "^5.0.0" + } + }, "parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", @@ -42553,6 +42736,11 @@ "integrity": "sha512-KddyFewCsO0j3+np81IQ+SweXLDnDQTs5s67BOnrYmYe/yNmUhttQyGsYzy8yUnoljGAQ9sl38YB4vH8ur7Y+w==", "dev": true }, + "q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==" + }, "qs": { "version": "6.11.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", @@ -43707,6 +43895,11 @@ "tslib": "^2.0.3" } }, + "snappyjs": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/snappyjs/-/snappyjs-0.6.1.tgz", + "integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg==" + }, "sockjs": { "version": "0.3.24", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz", @@ -44618,6 +44811,16 @@ "real-require": "^0.1.0" } }, + "thrift": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/thrift/-/thrift-0.11.0.tgz", + "integrity": "sha512-UpsBhOC45a45TpeHOXE4wwYwL8uD2apbHTbtBvkwtUU4dNwCjC7DpQTjw2Q6eIdfNtw+dKthdwq94uLXTJPfFw==", + "requires": { + "node-int64": "^0.4.0", + "q": "^1.5.0", + "ws": ">= 2.2.3" + } + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -45263,6 +45466,11 @@ "integrity": "sha512-Ii+sehpSfZy+At5nPdnyMhx78fEoPDkR2XW/zimHEL3MyGJQOCQ7WeP20jPYRz7ZCpcKLB21NxuXHF3bxjStBQ==", "devOptional": true }, + "varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==" + }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", diff --git a/package.json b/package.json index 787d3761..de6c6c2d 100644 --- a/package.json +++ b/package.json @@ -15,12 +15,14 @@ "duckdb": "^0.10.2", "express": "^4.19.2", "fake-indexeddb": "^5.0.1", + "fs": "0.0.1-security", "install": "^0.13.0", "lodash": "^4.17.21", "lodash.isnil": "^4.0.0", "loglevel": "^1.8.1", "next": "13.5.4", "npm": "^10.8.2", + "parquetjs": "0.11.2", "puppeteer": "^21.5.2", "react": "18.2.0", "react-dom": "18.2.0",