skip token count when local embedding

microsoft · Oct 31, 2024 · 6e72825 · 6e72825
1 parent 6c9296f
commit 6e72825
Showing 1 changed file with 8 additions and 4 deletions.
diff --git a/toolkit/AI/base_embedder.py b/toolkit/AI/base_embedder.py
@@ -94,10 +94,14 @@ def embed_store_one(
         if len(existing_embedding) > 0:
             return existing_embedding.get("vector")[0]
 
-        tokens = get_token_count(text)
-        if tokens > self.max_tokens:
-            text = text[: self.max_tokens]
-            logger.info("Truncated text to max tokens")
+        # error when local
+        try:
+            tokens = get_token_count(text)
+            if tokens > self.max_tokens:
+                text = text[: self.max_tokens]
+                logger.info("Truncated text to max tokens")
+        except:
+            pass
 
         try:
             embedding = self._generate_embedding(text)