feat: add bulk json register (#49)

* add: bulk register

* add: test

* bump: version

* fix: test

Author: vpbs2

benchmarking/benchmarking-app/src/app/constants.ts

@@ -31,8 +31,8 @@ export const TEST_QUERIES = [
         group_by_query.hvfhs_license_num = full_query.hvfhs_license_num
     LIMIT 1
       `,
-  'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxijson',
-  'SELECT * FROM taxijson WHERE price >= 1.0005812645 LIMIT 100',
-  'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxijson GROUP BY order_count',
-  'SELECT * as total_count FROM taxijson ORDER BY seconds_in_bucket LIMIT 100',
+  'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxijson.parquet',
+  'SELECT * FROM taxijson.parquet WHERE price >= 1.0005812645 LIMIT 100',
+  'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxijson.parquet GROUP BY order_count',
+  'SELECT * as total_count FROM taxijson.parquet ORDER BY seconds_in_bucket LIMIT 100',
 ];

benchmarking/benchmarking-app/src/app/file-loader/file-loader.tsx

@@ -25,7 +25,7 @@ export const FileLoader = ({ children }: { children: JSX.Element }) => {
       await fileManager.registerJSON({
         json: TAXI_JSON_DATA,
         tableName: 'taxijson',
-        fileName: 'taxi.json',
+        fileName: 'taxijson.parquet',
       });
 
       setIsFileLoader(true);

meerkat-dbm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-dbm",
-  "version": "0.0.14",
+  "version": "0.0.141",
   "dependencies": {
     "tslib": "^2.3.0",
     "@duckdb/duckdb-wasm": "^1.28.0",

meerkat-dbm/src/dbm/dbm.spec.ts

@@ -29,6 +29,12 @@ export class MockFileManager implements FileManagerType {
     this.tables[prop.tableName].files.push(prop);
   }
 
+  async bulkRegisterJSON(props: FileJsonStore[]): Promise<void> {
+    for (const prop of props) {
+      await this.registerJSON(prop);
+    }
+  }
+
   async registerJSON(prop: FileJsonStore): Promise<void> {
     const { json, ...fileData } = prop;
 

meerkat-dbm/src/file-manager/file-manager-type.ts

@@ -51,6 +51,15 @@ export interface FileManagerType {
    */
   registerFileBuffer: (props: FileBufferStore) => Promise<void>;
 
+  /**
+   * @description
+   * Registers multiple JSON files in the file manager.
+   * It converts JSON objects to Uint8Arrays by writing them to Parquet files in a DuckDB database and registers them.
+   * Also emits an event with the time taken for the conversion.
+   * @param props - An array of FileJsonStore objects.
+   */
+  bulkRegisterJSON: (props: FileJsonStore[]) => Promise<void>;
+
   /**
    * @description
    * Registers a single JSON file in the file manager.

meerkat-dbm/src/file-manager/indexed-db/__tests__/indexed-db-file-manager.spec.ts

@@ -216,21 +216,58 @@ describe('IndexedDBFileManager', () => {
   });
 
   it('should register JSON data', async () => {
-    const fileJson = {
+    const jsonFile = {
       tableName: 'taxi-json',
       fileName: 'taxi-json.parquet',
       json: {
         test: 'test',
       },
     };
 
-    await fileManager.registerJSON(fileJson);
+    await fileManager.registerJSON(jsonFile);
 
     const tableData = await indexedDB.tablesKey.toArray();
     const fileBufferData = await indexedDB.files.toArray();
 
-    tableData.some((table) => table.tableName === fileJson.tableName);
-    fileBufferData.some((file) => file.fileName === fileJson.fileName);
+    expect(
+      tableData.some((table) => table.tableName === jsonFile.tableName)
+    ).toBe(true);
+
+    expect(
+      fileBufferData.some((file) => file.fileName === jsonFile.fileName)
+    ).toBe(true);
+  });
+
+  it('should register multiple JSON data', async () => {
+    const jsonFiles = [
+      {
+        tableName: 'taxi-json-bulk',
+        fileName: 'taxi-json1.parquet',
+        json: {
+          test: 'test',
+        },
+      },
+      {
+        tableName: 'taxi-json-bulk',
+        fileName: 'taxi-json2.parquet',
+        json: {
+          test: 'test',
+        },
+      },
+    ];
+
+    await fileManager.bulkRegisterJSON(jsonFiles);
+
+    const tableData = await indexedDB.tablesKey.toArray();
+    const fileBufferData = await indexedDB.files.toArray();
+
+    expect(
+      tableData.some((table) => table.tableName === jsonFiles[0].tableName)
+    ).toBe(true);
+
+    expect(fileBufferData.map((file) => file.fileName)).toEqual(
+      expect.arrayContaining(jsonFiles.map((file) => file.fileName))
+    );
   });
 });
 

meerkat-dbm/src/file-manager/indexed-db/indexed-db-file-manager.ts

@@ -121,6 +121,27 @@ export class IndexedDBFileManager implements FileManagerType {
       });
   }
 
+  async bulkRegisterJSON(jsonData: FileJsonStore[]): Promise<void> {
+    const fileBuffers = await Promise.all(
+      jsonData.map(async (jsonFile) => {
+        const { json, tableName, ...fileData } = jsonFile;
+
+        const bufferData = await getBufferFromJSON({
+          instanceManager: this.instanceManager,
+          json: json,
+          tableName,
+          logger: this.logger,
+          onEvent: this.onEvent,
+          metadata: jsonFile.metadata,
+        });
+
+        return { buffer: bufferData, tableName, ...fileData };
+      })
+    );
+
+    await this.bulkRegisterFileBuffer(fileBuffers);
+  }
+
   async registerJSON(jsonData: FileJsonStore): Promise<void> {
     const { json, tableName, ...fileData } = jsonData;
 

meerkat-dbm/src/file-manager/memory-file-manager.ts

@@ -44,6 +44,14 @@ export class MemoryDBFileManager implements FileManagerType {
     return db.registerFileBuffer(props.fileName, props.buffer);
   }
 
+  async bulkRegisterJSON(jsonData: FileJsonStore[]): Promise<void> {
+    const promiseArr = jsonData.map((fileBuffer) =>
+      this.registerJSON(fileBuffer)
+    );
+
+    await Promise.all(promiseArr);
+  }
+
   async registerJSON(jsonData: FileJsonStore): Promise<void> {
     const { json, tableName, ...fileData } = jsonData;
 

meerkat-dbm/src/utils/get-buffer-from-json.ts

@@ -68,6 +68,9 @@ export const getBufferFromJSON = async ({
     metadata: { ...metadata, json },
   });
 
+  // Drop the table.
+  await connection.query(`DROP TABLE ${tableName};`);
+
   await db.registerEmptyFileBuffer(jsonFileName);
 
   await connection.close();