From 5adf34b102a542814cc17b2460da900cdd88e294 Mon Sep 17 00:00:00 2001 From: DARREN OBERST Date: Tue, 5 Mar 2024 09:22:23 -0500 Subject: [PATCH] update faiss optional import --- llmware/embeddings.py | 23 +++++++++++++++++++---- llmware/exceptions.py | 3 +-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/llmware/embeddings.py b/llmware/embeddings.py index 5382ad43..bc8d484c 100644 --- a/llmware/embeddings.py +++ b/llmware/embeddings.py @@ -20,7 +20,6 @@ import os -import faiss import logging import numpy as np import re @@ -30,6 +29,11 @@ from pymilvus import connections, utility, FieldSchema, CollectionSchema, DataType, Collection from pymongo import MongoClient +try: + import faiss +except ImportError: + pass + # note: update- adding psycopg and postgres to core llmware package in version 0.2.0 try: from pgvector.psycopg import register_vector @@ -636,9 +640,20 @@ def create_new_embedding(self, doc_ids=None, batch_size=100): if not self.index: if os.path.exists(self.embedding_file_path): - self.index = faiss.read_index(self.embedding_file_path) - else: - self.index = faiss.IndexFlatL2(self.embedding_dims) + + # shifted faiss to optional dependency + # note: there may be an edge case where this faiss command would fail even with + # library installed, but we throw dependency not installed error as most likely cause + + try: + self.index = faiss.read_index(self.embedding_file_path) + except: + raise DependencyNotInstalledException("faiss-cpu") + else: + try: + self.index = faiss.IndexFlatL2(self.embedding_dims) + except: + raise DependencyNotInstalledException("faiss-cpu") # get cursor for text collection with blocks requiring embedding all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids) diff --git a/llmware/exceptions.py b/llmware/exceptions.py index 523068d3..2b2b66e4 100644 --- a/llmware/exceptions.py +++ b/llmware/exceptions.py @@ -92,8 +92,7 @@ def __init__(self, prompt_name): class DependencyNotInstalledException(LLMWareException): def __init__(self, required_library_dependency): - message = f"'{required_library_dependency}' needs to be installed to use this function. Please refer to the " \ - f"documentation with any questions. " + message = f"'{required_library_dependency}' needs to be installed to use this function." super().__init__(message)