Skip to content

Commit

Permalink
update faiss optional import
Browse files Browse the repository at this point in the history
  • Loading branch information
DARREN OBERST authored and DARREN OBERST committed Mar 5, 2024
1 parent cfb2176 commit 5adf34b
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 6 deletions.
23 changes: 19 additions & 4 deletions llmware/embeddings.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@


import os
import faiss
import logging
import numpy as np
import re
Expand All @@ -30,6 +29,11 @@
from pymilvus import connections, utility, FieldSchema, CollectionSchema, DataType, Collection
from pymongo import MongoClient

try:
import faiss
except ImportError:
pass

# note: update- adding psycopg and postgres to core llmware package in version 0.2.0
try:
from pgvector.psycopg import register_vector
Expand Down Expand Up @@ -636,9 +640,20 @@ def create_new_embedding(self, doc_ids=None, batch_size=100):

if not self.index:
if os.path.exists(self.embedding_file_path):
self.index = faiss.read_index(self.embedding_file_path)
else:
self.index = faiss.IndexFlatL2(self.embedding_dims)

# shifted faiss to optional dependency
# note: there may be an edge case where this faiss command would fail even with
# library installed, but we throw dependency not installed error as most likely cause

try:
self.index = faiss.read_index(self.embedding_file_path)
except:
raise DependencyNotInstalledException("faiss-cpu")
else:
try:
self.index = faiss.IndexFlatL2(self.embedding_dims)
except:
raise DependencyNotInstalledException("faiss-cpu")

# get cursor for text collection with blocks requiring embedding
all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids)
Expand Down
3 changes: 1 addition & 2 deletions llmware/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,7 @@ def __init__(self, prompt_name):
class DependencyNotInstalledException(LLMWareException):

def __init__(self, required_library_dependency):
message = f"'{required_library_dependency}' needs to be installed to use this function. Please refer to the " \
f"documentation with any questions. "
message = f"'{required_library_dependency}' needs to be installed to use this function."
super().__init__(message)


Expand Down

0 comments on commit 5adf34b

Please sign in to comment.