feat: indexeddb file manager (#30)

* add: indexed dbm layer

* add: dexie

* chore: remove idb

* update: changes

* add: file manager test

* add: benchmarking test

* update: comments

* add: return

* update: dexie

* remove: comment

* address: comments

* add: tables map

* update: comment

* update: use tables

* fix: test

* bump version

Author: vpbs2

benchmarking/benchmarking-app/src/app/app.tsx

@@ -1,4 +1,5 @@
 import { Route, BrowserRouter as Router, Routes } from 'react-router-dom';
+import { IndexedDBMProvider } from './dbm-context/indexed-dbm-context';
 import { MemoryDBMProvider } from './dbm-context/memory-dbm-context';
 import { RawDBMProvider } from './dbm-context/raw-dbm-context';
 import { FileLoader } from './file-loader/file-loader';
@@ -35,6 +36,19 @@ export function App() {
             </div>
           }
         />
+        <Route
+          path="/indexed-dbm"
+          element={
+            <div>
+              <h1>IndexedDB DuckDB</h1>
+              <IndexedDBMProvider>
+                <FileLoader>
+                  <QueryBenchmarking />
+                </FileLoader>
+              </IndexedDBMProvider>
+            </div>
+          }
+        />
       </Routes>
     </Router>
   );

benchmarking/benchmarking-app/src/app/benchmarking-tests/dbm-benchmarking.spec.ts

@@ -7,6 +7,8 @@ describe('Benchmarking DBMs', () => {
   let browser;
   let appProcess;
 
+  let totalTimeForMemoryDB: number;
+
   beforeAll(async () => {
     appProcess = spawn('npx', ['nx', 'serve', 'benchmarking-app'], {
       stdio: 'inherit',
@@ -58,7 +60,7 @@ describe('Benchmarking DBMs', () => {
     /**
      * Get the total time as number
      */
-    const totalTimeForMemoryDB = await page.$eval('#total_time', (el) =>
+    totalTimeForMemoryDB = await page.$eval('#total_time', (el) =>
       Number(el.textContent)
     );
 
@@ -70,6 +72,27 @@ describe('Benchmarking DBMs', () => {
     expect(totalTimeForRawDB).toBeLessThan(totalTimeForMemoryDB * 1.1);
   }, 220000);
 
+  it('Benchmark indexed dbm duckdb', async () => {
+    await page.goto('http://localhost:4200/indexed-dbm');
+    /**
+     * wait for total time to be render
+     */
+    await page.waitForSelector('#total_time', { timeout: 300000 });
+    /**
+     * Get the total time as number
+     */
+    const totalTimeForIndexedDBM = await page.$eval('#total_time', (el) =>
+      Number(el.textContent)
+    );
+
+    console.info('totalTimeForIndexedDBM', totalTimeForIndexedDBM);
+
+    /**
+     * The total diff between indexed dbm and memory dbm should be less than 30%
+     */
+    expect(totalTimeForIndexedDBM).toBeLessThan(totalTimeForMemoryDB * 1.3);
+  }, 300000);
+
   afterAll(async () => {
     await browser.close();
     appProcess.kill('SIGTERM');

benchmarking/benchmarking-app/src/app/dbm-context/indexed-dbm-context.tsx

@@ -0,0 +1,48 @@
+import { DBM, IndexedDBFileManager } from '@devrev/meerkat-dbm';
+import React, { useState } from 'react';
+import { DBMContext } from '../hooks/dbm-context';
+import { useClassicEffect } from '../hooks/use-classic-effect';
+import { useAsyncDuckDB } from './use-async-duckdb';
+
+export const IndexedDBMProvider = ({ children }: { children: JSX.Element }) => {
+  const fileManagerRef = React.useRef<IndexedDBFileManager | null>(null);
+  const [dbm, setdbm] = useState<DBM | null>(null);
+
+  const dbState = useAsyncDuckDB();
+
+  useClassicEffect(() => {
+    if (!dbState) {
+      return;
+    }
+    fileManagerRef.current = new IndexedDBFileManager({
+      db: dbState,
+      fetchTableFileBuffers: async (table) => {
+        return [];
+      },
+    });
+
+    fileManagerRef.current.initializeDB();
+
+    const dbm = new DBM({
+      db: dbState,
+      fileManager: fileManagerRef.current,
+    });
+
+    setdbm(dbm);
+  }, [dbState]);
+
+  if (!dbm || !fileManagerRef.current) {
+    return <div>Loading...</div>;
+  }
+
+  return (
+    <DBMContext.Provider
+      value={{
+        dbm,
+        fileManager: fileManagerRef.current,
+      }}
+    >
+      {children}
+    </DBMContext.Provider>
+  );
+};

benchmarking/benchmarking-app/src/app/file-loader/file-loader.tsx

@@ -20,6 +20,7 @@ export const FileLoader = ({ children }: { children: JSX.Element }) => {
         fileName: 'taxi.parquet',
         buffer: fileBufferView,
       });
+
       setIsFileLoader(true);
     })();
   }, []);

benchmarking/benchmarking-app/src/app/query-benchmarking/query-benchmarking.tsx

@@ -21,30 +21,30 @@ export const QueryBenchmarking = () => {
       'SELECT * as total_count FROM taxi.parquet ORDER BY bcf LIMIT 100',
       `
       WITH group_by_query AS (
-        SELECT 
-            hvfhs_license_num, 
-            COUNT(*) 
-        FROM 
-            taxi.parquet 
-        GROUP BY 
+        SELECT
+            hvfhs_license_num,
+            COUNT(*)
+        FROM
+            taxi.parquet
+        GROUP BY
             hvfhs_license_num
-    ), 
-    
+    ),
+
     full_query AS (
-        SELECT 
-            * 
-        FROM 
+        SELECT
+            *
+        FROM
             taxi.parquet
-    ) 
-    
-    SELECT 
-        COUNT(*) 
-    FROM 
-        group_by_query 
-    LEFT JOIN 
-        full_query 
-    ON 
-        group_by_query.hvfhs_license_num = full_query.hvfhs_license_num 
+    )
+
+    SELECT
+        COUNT(*)
+    FROM
+        group_by_query
+    LEFT JOIN
+        full_query
+    ON
+        group_by_query.hvfhs_license_num = full_query.hvfhs_license_num
     LIMIT 1
       `,
     ];
@@ -56,7 +56,7 @@ export const QueryBenchmarking = () => {
       const eachQueryStart = performance.now();
 
       const promiseObj = dbm
-        .queryWithTableNames(testQueries[i], ['taxi.parquet'])
+        .queryWithTableNames(testQueries[i], ['taxi'])
         .then((results) => {
           const end = performance.now();
           const time = end - eachQueryStart;

examples/meerkat-node-example/data/fhvhv_tripdata_2023-01.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb066a3fe0770e44dde67cfc43e8b964b7b3ca7f2c90dc35d2d57cb7e41629b8
+size 473816636

meerkat-dbm/package.json

@@ -1,9 +1,10 @@
 {
   "name": "@devrev/meerkat-dbm",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "dependencies": {
     "tslib": "^2.3.0",
-    "@duckdb/duckdb-wasm": "^1.28.0"
+    "@duckdb/duckdb-wasm": "^1.28.0",
+    "dexie": "^3.2.4"
   },
   "repository": {
     "type": "git",

meerkat-dbm/src/file-manager/file-manager-type.ts

@@ -12,7 +12,7 @@ export interface FileBufferStore {
 export interface FileManagerType {
   bulkRegisterFileBuffer: (props: FileBufferStore[]) => Promise<void>;
   registerFileBuffer: (props: FileBufferStore) => Promise<void>;
-  getFileBuffer: (name: string) => Promise<Uint8Array>;
+  getFileBuffer: (name: string) => Promise<Uint8Array | undefined>;
   mountFileBufferByTableNames: (tableName: string[]) => Promise<void>;
   unmountFileBufferByTableNames: (tableName: string[]) => Promise<void>;
 }
@@ -21,3 +21,30 @@ export interface FileManagerConstructorOptions {
   fetchTableFileBuffers: (tableName: string) => Promise<FileBufferStore[]>;
   db: AsyncDuckDB;
 }
+
+export const FILE_TYPES = {
+  PARQUET: 'parquet',
+} as const;
+
+export type FileType = (typeof FILE_TYPES)[keyof typeof FILE_TYPES];
+
+export interface Table {
+  tableName: string;
+  files: FileData[];
+  totalSize?: number;
+  metadata?: object;
+}
+
+export interface FileData {
+  fileName: string;
+  fileType?: FileType;
+  size?: number;
+  staleTime?: number;
+  cacheTime?: number;
+  metadata?: object;
+}
+
+export interface File {
+  fileName: string;
+  buffer: Uint8Array;
+}

meerkat-dbm/src/file-manager/indexed-db/duckdb-database.ts

@@ -0,0 +1,24 @@
+import Dexie from 'dexie';
+import { File, Table } from '../file-manager-type';
+
+/**
+ * We don't need to specify the buffer field for index
+ * https://dexie.org/docs/Version/Version.stores()#warning
+ */
+
+export class DuckDBDatabase extends Dexie {
+  tablesKey: Dexie.Table<Table, string>;
+  files: Dexie.Table<File, string>;
+
+  constructor() {
+    super('DuckDBDatabase');
+
+    this.version(1).stores({
+      tablesKey: '&tableName',
+      files: '&fileName',
+    });
+
+    this.tablesKey = this.table('tablesKey');
+    this.files = this.table('files');
+  }
+}