Update github action

Signed-off-by: Jay Wang <[email protected]>
poloclub · Feb 4, 2024 · cdb18bc · cdb18bc
1 parent d711874
commit cdb18bc
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 32 deletions.
diff --git a/examples/rag-playground/.eslintrc.cjs b/examples/rag-playground/.eslintrc.cjs
@@ -52,8 +52,6 @@ module.exports = {
     // ],
     'no-constant-condition': ['error', { checkLoops: false }],
     'prettier/prettier': 2,
-    '@typescript-eslint/no-floating-promises': 'off',
-    '@typescript-eslint/no-unsafe-return': 'off',
     '@typescript-eslint/ban-ts-comment': 'off',
     '@typescript-eslint/restrict-template-expressions': 'off',
     '@typescript-eslint/no-non-null-assertion': 'off',

diff --git a/examples/rag-playground/src/workers/mememo-worker.ts b/examples/rag-playground/src/workers/mememo-worker.ts
@@ -64,6 +64,11 @@ const flexIndex: Flexsearch.Index = new Flexsearch.Index({
 }) as Flexsearch.Index;
 let workerDatasetName = 'my-dataset';
 let documentDBPromise: Promise<IDBPDatabase<string>> | null = null;
+const hnswIndex = new HNSW<string>({
+  distanceFunction: 'cosine',
+  seed: 123,
+  useIndexedDB: false
+});
 
 //==========================================================================||
 //                                Functions                                 ||
@@ -86,7 +91,10 @@ self.onmessage = (e: MessageEvent<MememoWorkerMessage>) => {
 
     case 'startLexicalSearch': {
       const { query, limit, requestID } = e.data.payload;
-      searchPoint(query, limit, requestID);
+      searchPoint(query, limit, requestID).then(
+        () => {},
+        () => {}
+      );
       break;
     }
 
@@ -112,33 +120,36 @@ const startLoadCompressedData = (url: string, datasetName: string) => {
     }
   });
 
-  fetch(url).then(async response => {
-    if (!response.ok) {
-      console.error('Failed to load data', response);
-      return;
-    }
+  fetch(url).then(
+    async response => {
+      if (!response.ok) {
+        console.error('Failed to load data', response);
+        return;
+      }
 
-    const reader = response.body
-      ?.pipeThrough(new DecompressionStream('gzip'))
-      ?.pipeThrough(new TextDecoderStream())
-      ?.pipeThrough(splitStreamTransform('\n'))
-      ?.pipeThrough(parseJSONTransform())
-      ?.getReader();
-
-    while (true && reader !== undefined) {
-      const result = await reader.read();
-      const point = result.value as DocumentRecordStreamData;
-      const done = result.done;
-
-      if (done) {
-        timeit('Stream data', DEV_MODE);
-        pointStreamFinished();
-        break;
-      } else {
-        await processPointStream(point);
+      const reader = response.body
+        ?.pipeThrough(new DecompressionStream('gzip'))
+        ?.pipeThrough(new TextDecoderStream())
+        ?.pipeThrough(splitStreamTransform('\n'))
+        ?.pipeThrough(parseJSONTransform())
+        ?.getReader();
+
+      while (true && reader !== undefined) {
+        const result = await reader.read();
+        const point = result.value as DocumentRecordStreamData;
+        const done = result.done;
+
+        if (done) {
+          timeit('Stream data', DEV_MODE);
+          pointStreamFinished();
+          break;
+        } else {
+          await processPointStream(point);
+        }
       }
-    }
-  });
+    },
+    () => {}
+  );
 };
 
 /**
@@ -161,6 +172,8 @@ const processPointStream = async (point: DocumentRecordStreamData) => {
   pendingDataPoints.push(documentPoint);
   flexIndex.add(documentPoint.id, documentPoint.text);
   await documentDB.put(workerDatasetName, documentPoint.text, documentPoint.id);
+  await hnswIndex.insert(String(documentPoint.id), documentPoint.embedding);
+
   loadedPointCount += 1;
 
   // Notify the main thread if we have load enough data
@@ -175,9 +188,9 @@ const processPointStream = async (point: DocumentRecordStreamData) => {
       }
     };
 
-    await new Promise<void>(resolve => {
-      setTimeout(resolve, 100);
-    });
+    // await new Promise<void>(resolve => {
+    //   setTimeout(resolve, 100);
+    // });
 
     postMessage(result);
 

diff --git a/package.json b/package.json
@@ -36,7 +36,7 @@
   "scripts": {
     "test": "vitest test/mememo.test.ts",
     "test:browser": "vitest -c vitest.config.browser.ts test/mememo.browser.test.ts",
-    "test:run": "vitest run test/mememo.test.ts && test:run:browser",
+    "test:run": "vitest run test/mememo.test.ts && npnm run test:run:browser",
     "test:run:browser": "vitest run -c vitest.config.browser.ts test/mememo.browser.test.ts",
     "coverage": "vitest run --coverage && c8 report && pnpm run coverage:badge",
     "coverage:badge": "pnpx make-coverage-badge --output-path ./imgs/coverage-badge.svg",