Adding puppeteer benchmarking test between raw duckdb & sequence memory DBM (#26)

* Adding puppeteer based tests

* Adding tests

* Run tests in CI

* Run now

* Git clone with large files

* Remove unwanted stuff

* Remove unwanted stuff

* Remove cypress

* Run

* Increase timeout

* Adding raw duckdb runner

* Adding test

Author: itsTalwar

.github/workflows/build-test.yml

@@ -10,6 +10,8 @@ jobs:
         runs-on: ubuntu-latest
         steps:
         - uses: actions/checkout@v2
+          with:
+            lfs: true
         - run: npm install
         - run: npx nx run-many --target=build --all --parallel
         - run: npx nx run-many --target=test --all --parallel

benchmarking/benchmarking-app/src/app/app.tsx

@@ -1,16 +1,32 @@
 import { Route, BrowserRouter as Router, Routes } from 'react-router-dom';
 import { MemoryDBMProvider } from './dbm-context/memory-dbm-context';
+import { RawDBMProvider } from './dbm-context/raw-dbm-context';
 import { FileLoader } from './file-loader/file-loader';
 import { QueryBenchmarking } from './query-benchmarking/query-benchmarking';
 
 export function App() {
   return (
     <Router>
       <Routes>
+        <Route
+          path="/raw-dbm"
+          element={
+            <div>
+              <h1>Raw DuckDB</h1>
+              <RawDBMProvider>
+                <FileLoader>
+                  <QueryBenchmarking />
+                </FileLoader>
+              </RawDBMProvider>
+            </div>
+          }
+        />
         <Route
           path="/memory-dbm"
           element={
             <div>
+              <h1>In Memory Sequence DuckDB</h1>
+
               <MemoryDBMProvider>
                 <FileLoader>
                   <QueryBenchmarking />

benchmarking/benchmarking-app/src/app/benchmarking-tests/dbm-benchmarking.spec.ts

@@ -0,0 +1,77 @@
+import axios from 'axios';
+import { spawn } from 'child_process';
+import * as puppeteer from 'puppeteer';
+
+describe('Benchmarking DBMs', () => {
+  let page;
+  let browser;
+  let appProcess;
+
+  beforeAll(async () => {
+    appProcess = spawn('npx', ['nx', 'serve', 'benchmarking-app'], {
+      stdio: 'inherit',
+    });
+
+    // Wait for the server to start
+    let serverStarted = false;
+    while (!serverStarted) {
+      try {
+        const response = await axios.get('http://localhost:4200'); // Replace with your server's URL
+        if (response.status === 200) {
+          serverStarted = true;
+        } else {
+          await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before retrying
+        }
+      } catch (error) {
+        await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before retrying
+      }
+    }
+
+    browser = await puppeteer.launch({
+      headless: 'new',
+    });
+    page = await browser.newPage();
+  }, 30000);
+
+  it('Benchmark raw duckdb with memory sequence duckdb', async () => {
+    await page.goto('http://localhost:4200/raw-dbm');
+    /**
+     * wait for total time to be render
+     */
+    await page.waitForSelector('#total_time', { timeout: 120000 });
+    /**
+     * Get the total time as number
+     */
+    const totalTimeForRawDB = await page.$eval('#total_time', (el) =>
+      Number(el.textContent)
+    );
+
+    console.info('totalTimeForRawDB', totalTimeForRawDB);
+
+    await page.goto('http://localhost:4200/memory-dbm');
+
+    /**
+     * wait for total time to be render
+     */
+    await page.waitForSelector('#total_time', { timeout: 120000 });
+
+    /**
+     * Get the total time as number
+     */
+    const totalTimeForMemoryDB = await page.$eval('#total_time', (el) =>
+      Number(el.textContent)
+    );
+
+    console.info('totalTimeForMemoryDB', totalTimeForMemoryDB);
+
+    /**
+     * The total diff between the two DBs should be less than 10%
+     */
+    expect(totalTimeForRawDB).toBeLessThan(totalTimeForMemoryDB * 1.1);
+  }, 220000);
+
+  afterAll(async () => {
+    await browser.close();
+    appProcess.kill('SIGTERM');
+  });
+});

benchmarking/benchmarking-app/src/app/dbm-context/raw-dbm-context.tsx

@@ -0,0 +1,50 @@
+import { DBM, FileManagerType, MemoryDBFileManager } from '@devrev/meerkat-dbm';
+import React, { useState } from 'react';
+import { DBMContext } from '../hooks/dbm-context';
+import { useClassicEffect } from '../hooks/use-classic-effect';
+import { useAsyncDuckDB } from './use-async-duckdb';
+
+export const RawDBMProvider = ({ children }: { children: JSX.Element }) => {
+  const fileManagerRef = React.useRef<FileManagerType | null>(null);
+  const [dbm, setdbm] = useState<DBM | null>(null);
+
+  const dbState = useAsyncDuckDB();
+
+  useClassicEffect(() => {
+    if (!dbState) {
+      return;
+    }
+    fileManagerRef.current = new MemoryDBFileManager({
+      db: dbState,
+      fetchTableFileBuffers: async (table) => {
+        return [];
+      },
+    });
+    const dbm = new DBM({
+      db: dbState,
+      fileManager: fileManagerRef.current,
+    });
+    /**
+     * Making the queryWithTableNames simply run the queries without sequence which is the default behavior
+     */
+    dbm.queryWithTableNames = async (query, tableNames) => {
+      return dbm.query(query);
+    };
+    setdbm(dbm);
+  }, [dbState]);
+
+  if (!dbm || !fileManagerRef.current) {
+    return <div>Loading...</div>;
+  }
+
+  return (
+    <DBMContext.Provider
+      value={{
+        dbm,
+        fileManager: fileManagerRef.current,
+      }}
+    >
+      {children}
+    </DBMContext.Provider>
+  );
+};

benchmarking/benchmarking-app/src/app/query-benchmarking/query-benchmarking.tsx

@@ -9,43 +9,91 @@ export const QueryBenchmarking = () => {
       time: number;
     }[]
   >([]);
+  const [totalTime, setTotalTime] = useState<number>(0);
   const { dbm } = useDBM();
 
   useClassicEffect(() => {
+    setTotalTime(0);
     const testQueries = [
-      'SELECT * as total_count FROM taxi.parquet ORDER BY bcf LIMIT 100',
       'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet',
       "SELECT * FROM taxi.parquet WHERE originating_base_num='B03404' LIMIT 100",
       'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet GROUP BY hvfhs_license_num',
+      'SELECT * as total_count FROM taxi.parquet ORDER BY bcf LIMIT 100',
+      `
+      WITH group_by_query AS (
+        SELECT 
+            hvfhs_license_num, 
+            COUNT(*) 
+        FROM 
+            taxi.parquet 
+        GROUP BY 
+            hvfhs_license_num
+    ), 
+    
+    full_query AS (
+        SELECT 
+            * 
+        FROM 
+            taxi.parquet
+    ) 
+    
+    SELECT 
+        COUNT(*) 
+    FROM 
+        group_by_query 
+    LEFT JOIN 
+        full_query 
+    ON 
+        group_by_query.hvfhs_license_num = full_query.hvfhs_license_num 
+    LIMIT 1
+      `,
     ];
 
     setOutput([]);
-
+    const promiseArr = [];
+    const start = performance.now();
     for (let i = 0; i < testQueries.length; i++) {
-      const start = performance.now();
+      const eachQueryStart = performance.now();
 
-      dbm
+      const promiseObj = dbm
         .queryWithTableNames(testQueries[i], ['taxi.parquet'])
         .then((results) => {
           const end = performance.now();
-          const time = end - start;
+          const time = end - eachQueryStart;
           setOutput((prev) => [
             ...prev,
             { queryName: `Query ${i} ---->`, time },
           ]);
         });
+
+      promiseArr.push(promiseObj);
     }
+    Promise.all(promiseArr).then(() => {
+      const end = performance.now();
+      const time = end - start;
+      setTotalTime(time);
+    });
   }, []);
 
   return (
     <div>
-      {output.map((o) => {
+      {output.map((o, i) => {
         return (
-          <div key={o.queryName}>
+          <div data-query={`${i}`} key={o.queryName}>
             {o.queryName} : {o.time}
           </div>
         );
       })}
+      {totalTime === 0 && (
+        <div>
+          <span>Query Running...</span>
+        </div>
+      )}
+      {totalTime > 0 && (
+        <div>
+          Total Time: <span id="total_time">{totalTime}</span>
+        </div>
+      )}
     </div>
   );
 };

benchmarking/benchmarking-app/tsconfig.app.json

@@ -6,7 +6,8 @@
       "node",
 
       "@nx/react/typings/cssmodule.d.ts",
-      "@nx/react/typings/image.d.ts"
+      "@nx/react/typings/image.d.ts",
+      "puppeteer"
     ]
   },
   "exclude": [

benchmarking/benchmarking-app/tsconfig.spec.json

@@ -3,7 +3,7 @@
   "compilerOptions": {
     "outDir": "../../dist/out-tsc",
     "module": "commonjs",
-    "types": ["jest", "node"]
+    "types": ["jest", "node", "puppeteer"]
   },
   "include": [
     "jest.config.ts",

examples/meerkat-node-example/tsconfig.app.json

@@ -3,7 +3,7 @@
   "compilerOptions": {
     "outDir": "../../dist/out-tsc",
     "module": "commonjs",
-    "types": ["node", "express"]
+    "types": ["node", "express", "puppeteer"]
   },
   "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"],
   "include": ["src/**/*.ts"]

examples/meerkat-node-example/tsconfig.spec.json

@@ -3,7 +3,7 @@
   "compilerOptions": {
     "outDir": "../../dist/out-tsc",
     "module": "commonjs",
-    "types": ["jest", "node"]
+    "types": ["jest", "node", "puppeteer"]
   },
   "include": [
     "jest.config.ts",