nlp calculations are also happening in a worker and give back the cor…

…rect results with the correct ids...

src/modules/nlp.ts

@@ -4,7 +4,6 @@ import {
   cat,
   mean,
   cos_sim,
-  dot,
   magnitude,
 } from '@xenova/transformers';
 import * as sw from 'stopword'; // Assuming this is the stopword library you are referring to

src/modules/nlp.worker.ts

@@ -1,27 +1,23 @@
-import { getVector, extractKeywords } from './nlp';
+import { getVector } from './nlp';
+
+export type nlpWorkerResult = { vector: number[] | undefined; id: number };
 
 self.onmessage = async ({
   data,
 }: {
   data: {
-    action: string;
-    txt: string;
+    text: string;
     modelName: string;
-    numKeywords?: number;
+    id: number;
   };
 }) => {
-  const { action, txt, modelName, numKeywords } = data;
-
+  const message: nlpWorkerResult = { vector: undefined, id: data.id };
   try {
-    if (action === 'getVector') {
-      const vector = await getVector(txt, modelName);
-      self.postMessage({ vector });
-    } else if (action === 'extractKeywords') {
-      const keywords = await extractKeywords(txt, modelName, numKeywords);
-      self.postMessage({ keywords });
-    }
+    const vector = await getVector(data.text, data.modelName);
+    message.vector = vector;
   } catch (error) {
     // Handle any errors here
-    self.postMessage({ error: (error as Error).message });
+    console.error(error);
   }
+  self.postMessage(message);
 };

src/modules/pyodide.worker.ts

@@ -1,13 +1,12 @@
 import { loadPyodide, PyProxy, type PyodideInterface } from 'pyodide';
 import { PythonScriptResult, executeScript } from './pyodide';
-import { unknown } from 'zod';
 
 //declare const self: ServiceWorkerGlobalScope
 
 // Setup your project to serve `py-worker.js`. You should also serve
 // `pyodide.js`, and all its associated `.asm.js`, `.json`,
 // and `.wasm` files as well:
-importScripts('https://cdn.jsdelivr.net/pyodide/v0.24.1/full/pyodide.js');
+// importScripts('https://cdn.jsdelivr.net/pyodide/v0.24.1/full/pyodide.js');
 
 let pyodideEnv: PyodideInterface | undefined = undefined;
 

src/modules/taskManager.ts

@@ -15,11 +15,7 @@ import { deepMerge } from './utils';
 import { HierarchicalNSW } from 'hnswlib-wasm/dist/hnswlib-wasm';
 import { AsyncQueue } from './utils';
 import { loadOrCreateVectorStore } from './vectorSearch';
-import {
-  extractKeywords,
-  extractKeywordsFromText,
-  vectorizeText,
-} from './webWorkerApi';
+import { extractKeywords, vectorizeText } from './webWorkerApi';
 
 /**
  * Finds the root task of a given task.

src/modules/tools.ts

@@ -10,7 +10,7 @@ import {
 } from './types';
 import { z } from 'zod';
 import { toolStateType, FunctionCall, ParamType } from './types';
-import { asyncRun } from './webWorkerApi';
+import { asyncRunPython } from './webWorkerApi';
 
 export const vectorStore = useVectorStore();
 
@@ -137,7 +137,7 @@ tools.executePythonScript = {
     pythonScript: string;
   }): Promise<PythonScriptResult> => {
     console.log('execute python code...');
-    return await asyncRun(pythonScript);
+    return await asyncRunPython(pythonScript);
   },
   description: `Executes Python scripts for data processing, calculations, or library interactions, 
 ideal for data analysis, machine learning tasks, or custom algorithm execution.

src/modules/webWorkerApi.ts

@@ -1,69 +1,37 @@
 import type { PythonScriptResult } from './pyodide';
+import type { nlpWorkerResult } from './nlp.worker';
 
 const nlpWorker = new Worker(new URL('./nlp.worker.ts', import.meta.url));
 
-// Function to send data to the worker and receive the result
-export function vectorizeText(
-  text: string,
-  modelName: string
-): Promise<number[]> {
-  return new Promise((resolve, reject) => {
-    nlpWorker.onmessage = ({
-      data,
-    }: {
-      data: { error?: unknown; vector?: number[] };
-    }) => {
-      if (data.vector) {
-        resolve(data.vector);
-      } else if (data.error) {
-        reject(data.error);
-      }
-    };
+const nlpCallbacks: Record<number, (vector: number[] | undefined) => void> = {};
 
-    nlpWorker.onerror = (error) => {
-      reject(error);
-    };
-
-    console.log('calling worker with model:', modelName);
-    nlpWorker.postMessage({
-      action: 'getVector',
-      txt: text,
-      modelName: modelName,
-    });
-  });
-}
-
-export function extractKeywordsFromText(
-  text: string,
-  modelName: string,
-  numKeywords = 5
-): Promise<string[]> {
-  return new Promise((resolve, reject) => {
-    nlpWorker.onmessage = ({
-      data,
-    }: {
-      data: { error?: unknown; keywords?: string[] };
-    }) => {
-      if (data.keywords) {
-        resolve(data.keywords);
-      } else if (data.error) {
-        reject(data.error);
-      }
-    };
-
-    nlpWorker.onerror = (error) => {
-      reject(error);
-    };
+nlpWorker.onmessage = ({
+  data,
+}: {
+  data: nlpWorkerResult & { id: number };
+}) => {
+  const { id, ...res } = data;
+  const onSuccess = nlpCallbacks[id];
+  delete nlpCallbacks[id];
+  onSuccess(res.vector);
+};
 
-    console.log('calling worker for keywords with model:', modelName);
-    nlpWorker.postMessage({
-      action: 'extractKeywords',
-      txt: text,
-      modelName: modelName,
-      numKeywords: numKeywords,
+export const vectorizeText = (() => {
+  let id = 0; // identify a Promise
+  return (text: string, modelName: string) => {
+    // the id could be generated more carefully
+    id = (id + 1) % Number.MAX_SAFE_INTEGER;
+    return new Promise<number[] | undefined>((onSuccess) => {
+      nlpCallbacks[id] = onSuccess;
+      console.log('calling nlp webworker');
+      nlpWorker.postMessage({
+        text,
+        modelName,
+        id,
+      });
     });
-  });
-}
+  };
+})();
 
 const pyodideWorker = new Worker(
   new URL('./pyodide.worker.ts', import.meta.url)
@@ -81,7 +49,7 @@ pyodideWorker.onmessage = ({
   onSuccess(data.result);
 };
 
-export const asyncRun = (() => {
+export const asyncRunPython = (() => {
   let id = 0; // identify a Promise
   return (script: string, params?: unknown[]) => {
     // the id could be generated more carefully
@@ -110,7 +78,7 @@ def keywordsFunc(text: str):
   return keywords
 keywordsFunc
 `;
-  const res = await asyncRun(pythonScript, [text]);
+  const res = await asyncRunPython(pythonScript, [text]);
   console.log('keyword Result: ', res);
   try {
     const allKws = res.result as [string, number][];