Duckdb Benchmarking app (#25)

* Add parquet file and base setup

* Benchmarking queries

* Adding memory DBM

* Fix PR checks

* Remove the E2E app

Author: itsTalwar

.gitattributes

@@ -0,0 +1 @@
+*.parquet filter=lfs diff=lfs merge=lfs -text

benchmarking/benchmarking-app/.babelrc

@@ -0,0 +1,11 @@
+{
+  "presets": [
+    [
+      "@nx/react/babel",
+      {
+        "runtime": "automatic"
+      }
+    ]
+  ],
+  "plugins": []
+}

benchmarking/benchmarking-app/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "extends": ["plugin:@nx/react", "../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ]
+}

benchmarking/benchmarking-app/jest.config.ts

@@ -0,0 +1,11 @@
+/* eslint-disable */
+export default {
+  displayName: 'benchmarking-app',
+  preset: '../../jest.preset.js',
+  transform: {
+    '^(?!.*\\.(js|jsx|ts|tsx|css|json)$)': '@nx/react/plugins/jest',
+    '^.+\\.[tj]sx?$': ['babel-jest', { presets: ['@nx/react/babel'] }],
+  },
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
+  coverageDirectory: '../../coverage/benchmarking/benchmarking-app',
+};

benchmarking/benchmarking-app/project.json

@@ -0,0 +1,98 @@
+{
+  "name": "benchmarking-app",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "benchmarking/benchmarking-app/src",
+  "projectType": "application",
+  "targets": {
+    "build": {
+      "executor": "@nx/webpack:webpack",
+      "outputs": ["{options.outputPath}"],
+      "defaultConfiguration": "production",
+      "options": {
+        "compiler": "babel",
+        "outputPath": "dist/benchmarking/benchmarking-app",
+        "index": "benchmarking/benchmarking-app/src/index.html",
+        "baseHref": "/",
+        "main": "benchmarking/benchmarking-app/src/main.tsx",
+        "tsConfig": "benchmarking/benchmarking-app/tsconfig.app.json",
+        "assets": [
+          "benchmarking/benchmarking-app/src/favicon.ico",
+          "benchmarking/benchmarking-app/src/assets"
+        ],
+        "styles": ["benchmarking/benchmarking-app/src/styles.css"],
+        "scripts": [],
+        "isolatedConfig": true,
+        "webpackConfig": "benchmarking/benchmarking-app/webpack.config.js"
+      },
+      "configurations": {
+        "development": {
+          "extractLicenses": false,
+          "optimization": false,
+          "sourceMap": true,
+          "vendorChunk": true
+        },
+        "production": {
+          "fileReplacements": [
+            {
+              "replace": "benchmarking/benchmarking-app/src/environments/environment.ts",
+              "with": "benchmarking/benchmarking-app/src/environments/environment.prod.ts"
+            }
+          ],
+          "optimization": true,
+          "outputHashing": "all",
+          "sourceMap": false,
+          "namedChunks": false,
+          "extractLicenses": true,
+          "vendorChunk": false
+        }
+      }
+    },
+    "serve": {
+      "executor": "@nx/webpack:dev-server",
+      "defaultConfiguration": "development",
+      "options": {
+        "buildTarget": "benchmarking-app:build",
+        "hmr": true
+      },
+      "configurations": {
+        "development": {
+          "buildTarget": "benchmarking-app:build:development"
+        },
+        "production": {
+          "buildTarget": "benchmarking-app:build:production",
+          "hmr": false
+        }
+      }
+    },
+    "lint": {
+      "executor": "@nx/linter:eslint",
+      "outputs": ["{options.outputFile}"],
+      "options": {
+        "lintFilePatterns": [
+          "benchmarking/benchmarking-app/**/*.{ts,tsx,js,jsx}"
+        ]
+      }
+    },
+    "serve-static": {
+      "executor": "@nx/web:file-server",
+      "options": {
+        "buildTarget": "benchmarking-app:build"
+      }
+    },
+    "test": {
+      "executor": "@nx/jest:jest",
+      "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
+      "options": {
+        "jestConfig": "benchmarking/benchmarking-app/jest.config.ts",
+        "passWithNoTests": true
+      },
+      "configurations": {
+        "ci": {
+          "ci": true,
+          "codeCoverage": true
+        }
+      }
+    }
+  },
+  "tags": []
+}

benchmarking/benchmarking-app/src/app/app.module.css

@@ -0,0 +1 @@
+/* Your styles goes here. */

benchmarking/benchmarking-app/src/app/app.tsx

@@ -0,0 +1,27 @@
+import { Route, BrowserRouter as Router, Routes } from 'react-router-dom';
+import { MemoryDBMProvider } from './dbm-context/memory-dbm-context';
+import { FileLoader } from './file-loader/file-loader';
+import { QueryBenchmarking } from './query-benchmarking/query-benchmarking';
+
+export function App() {
+  return (
+    <Router>
+      <Routes>
+        <Route
+          path="/memory-dbm"
+          element={
+            <div>
+              <MemoryDBMProvider>
+                <FileLoader>
+                  <QueryBenchmarking />
+                </FileLoader>
+              </MemoryDBMProvider>
+            </div>
+          }
+        />
+      </Routes>
+    </Router>
+  );
+}
+
+export default App;

benchmarking/benchmarking-app/src/app/dbm-context/memory-dbm-context.tsx

@@ -0,0 +1,44 @@
+import { DBM, FileManagerType, MemoryDBFileManager } from '@devrev/meerkat-dbm';
+import React, { useState } from 'react';
+import { DBMContext } from '../hooks/dbm-context';
+import { useClassicEffect } from '../hooks/use-classic-effect';
+import { useAsyncDuckDB } from './use-async-duckdb';
+
+export const MemoryDBMProvider = ({ children }: { children: JSX.Element }) => {
+  const fileManagerRef = React.useRef<FileManagerType | null>(null);
+  const [dbm, setdbm] = useState<DBM | null>(null);
+
+  const dbState = useAsyncDuckDB();
+
+  useClassicEffect(() => {
+    if (!dbState) {
+      return;
+    }
+    fileManagerRef.current = new MemoryDBFileManager({
+      db: dbState,
+      fetchTableFileBuffers: async (table) => {
+        return [];
+      },
+    });
+    const dbm = new DBM({
+      db: dbState,
+      fileManager: fileManagerRef.current,
+    });
+    setdbm(dbm);
+  }, [dbState]);
+
+  if (!dbm || !fileManagerRef.current) {
+    return <div>Loading...</div>;
+  }
+
+  return (
+    <DBMContext.Provider
+      value={{
+        dbm,
+        fileManager: fileManagerRef.current,
+      }}
+    >
+      {children}
+    </DBMContext.Provider>
+  );
+};

benchmarking/benchmarking-app/src/app/dbm-context/use-async-duckdb.tsx

@@ -0,0 +1,36 @@
+import { DBM, FileManagerType } from '@devrev/meerkat-dbm';
+import * as duckdb from '@duckdb/duckdb-wasm';
+import { AsyncDuckDB } from '@duckdb/duckdb-wasm';
+import React, { useState } from 'react';
+import { useClassicEffect } from '../hooks/use-classic-effect';
+const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();
+
+export const DBMContext = React.createContext<{
+  dbm: DBM;
+  fileManager: FileManagerType;
+}>(null as any);
+
+export const useAsyncDuckDB = () => {
+  const [dbState, setdbState] = useState<AsyncDuckDB | null>(null);
+
+  useClassicEffect(() => {
+    (async () => {
+      const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
+
+      const worker_url = URL.createObjectURL(
+        new Blob([`importScripts("${bundle.mainWorker!}");`], {
+          type: 'text/javascript',
+        })
+      );
+
+      // Instantiate the asynchronus version of DuckDB-wasm
+      const worker = new Worker(worker_url);
+      const logger = new duckdb.ConsoleLogger();
+      const db = new duckdb.AsyncDuckDB(logger, worker);
+      await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
+      URL.revokeObjectURL(worker_url);
+      setdbState(db);
+    })();
+  }, []);
+  return dbState;
+};

benchmarking/benchmarking-app/src/app/file-loader/file-loader.tsx

@@ -0,0 +1,32 @@
+import axios from 'axios';
+import { useState } from 'react';
+import { useDBM } from '../hooks/dbm-context';
+import { useClassicEffect } from '../hooks/use-classic-effect';
+
+export const FileLoader = ({ children }: { children: JSX.Element }) => {
+  const { fileManager } = useDBM();
+  const [isFileLoader, setIsFileLoader] = useState<boolean>(false);
+  useClassicEffect(() => {
+    (async () => {
+      const file = await axios.get(
+        'http://localhost:4200/assets/data-sets/fhvhv_tripdata_2023-01.parquet',
+        { responseType: 'arraybuffer' }
+      );
+      const fileBuffer = file.data;
+      const fileBufferView = new Uint8Array(fileBuffer);
+
+      await fileManager.registerFileBuffer({
+        tableName: 'taxi',
+        fileName: 'taxi.parquet',
+        buffer: fileBufferView,
+      });
+      setIsFileLoader(true);
+    })();
+  }, []);
+
+  if (!isFileLoader) {
+    return <div>Loading file</div>;
+  }
+
+  return <>{children}</>;
+};