Adding llama2

common.env

@@ -1,4 +1,4 @@
-OPENAI_API_KEY=
+OPENAI_API_KEY=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
 
 # pinecone | qdrant
 STORE=qdrant

dj_backend_server/.gitignore

@@ -35,4 +35,5 @@ staticfiles
 pip-log.txt
 pip-delete-this-directory.txt
 website_data_sources/*
-venv
+venv
+open-llama-7B-open-instruct.ggmlv3.q4_K_M.bin

dj_backend_server/.vscode/launch.json

@@ -19,7 +19,6 @@
             "name": "Python: Celery Workers",
             "type": "python",
             "request": "launch",
-            // "program": "${workspaceFolder}/path_to_celery_executable",
             "module": "celery",
             "args": [
                 "-A",

dj_backend_server/api/enums/embedding_type.py

@@ -5,4 +5,5 @@ class EmbeddingProvider(Enum):
     OPENAI = "openai"
     BARD = "bard"
     azure = "azure"
+    llama2 = "llama2"
 

dj_backend_server/api/utils/get_embeddings.py

@@ -3,10 +3,9 @@
 import os
 from dotenv import load_dotenv
 from langchain.embeddings.base import Embeddings
+from langchain.embeddings import LlamaCppEmbeddings
 
 load_dotenv()
-
-# https://github.com/easonlai/azure_openai_langchain_sample/blob/main/chat_with_pdf.ipynb
 import os
 
 
@@ -37,6 +36,10 @@ def get_openai_embedding():
 
     return OpenAIEmbeddings(openai_api_key=openai_api_key, chunk_size=1)
 
+def get_llama2_embedding():
+    """Gets embeddings using the llama2 embedding provider."""
+    return LlamaCppEmbeddings(model_path="open-llama-7B-open-instruct.ggmlv3.q4_K_M.bin")
+
 def choose_embedding_provider():
     """Chooses and returns the appropriate embedding provider instance."""
     embedding_provider = get_embedding_provider()
@@ -46,6 +49,10 @@ def choose_embedding_provider():
 
     elif embedding_provider == EmbeddingProvider.OPENAI.value:
         return get_openai_embedding()
+
+    elif embedding_provider == EmbeddingProvider.llama2.value:
+        return get_llama2_embedding()
+
 
     else:
         available_providers = ", ".join([service.value for service in EmbeddingProvider])

dj_backend_server/api/utils/get_openai_llm.py

@@ -1,8 +1,32 @@
 from langchain.llms import AzureOpenAI, OpenAI
 import os
 from dotenv import load_dotenv
-
+from langchain.llms import LlamaCpp
 load_dotenv()
+from langchain import PromptTemplate, LLMChain
+from langchain.callbacks.manager import CallbackManager
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+
+
+def get_llama_llm():
+    n_gpu_layers = 1  # Metal set to 1 is enough.
+    n_batch = 512  # Should be between 1 and n_ctx, consider the amount of RAM of your Apple Silicon Chip.
+
+    # Callbacks support token-wise streaming
+    callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
+    llm = LlamaCpp(
+        model_path="open-llama-7B-open-instruct.ggmlv3.q4_K_M.bin",
+        n_gpu_layers=n_gpu_layers,
+        n_batch=n_batch,
+        n_ctx=4096,
+        f16_kv=True,  # MUST set to True, otherwise you will run into problem after a couple of calls
+        callback_manager=callback_manager,
+        verbose=True,
+        temperature=0.2,
+        use_mmap=True
+    )
+
+    return llm
 
 # Azure OpenAI Language Model client
 def get_azure_openai_llm():
@@ -43,7 +67,8 @@ def get_llm():
 
     clients = {
         'azure': get_azure_openai_llm,
-        'openai': get_openai_llm
+        'openai': get_openai_llm,
+        'llama2': get_llama_llm
     }
 
     api_type = os.environ.get('OPENAI_API_TYPE')

dj_backend_server/requirements.txt

@@ -1,12 +1,17 @@
+absl-py==1.4.0
 aiohttp==3.8.5
 aiosignal==1.3.1
 amqp==5.1.1
 anyio==3.7.1
 asgiref==3.7.2
+astunparse==1.6.3
 async-timeout==4.0.2
 attrs==23.1.0
 beautifulsoup4==4.12.2
 billiard==4.1.0
+blis==0.7.10
+cachetools==5.3.1
+catalogue==2.0.9
 celery==5.3.1
 certifi==2023.5.7
 cffi==1.15.1
@@ -16,42 +21,81 @@ click==8.1.6
 click-didyoumean==0.3.0
 click-plugins==1.1.1
 click-repl==0.3.0
+cloud-tpu-client==0.10
+confection==0.1.1
 cryptography==41.0.3
+cymem==2.0.7
 dataclasses-json==0.5.14
+diskcache==5.6.1
 Django==4.2.3
 dnspython==2.4.1
 exceptiongroup==1.1.2
+filelock==3.12.2
+flatbuffers==23.5.26
 frozenlist==1.4.0
+fsspec==2023.6.0
+gast==0.4.0
+google-api-core==1.34.0
+google-api-python-client==1.8.0
+google-auth==2.22.0
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==1.0.0
+google-pasta==0.2.0
+googleapis-common-protos==1.60.0
 grpcio==1.56.2
 grpcio-tools==1.56.2
 h11==0.14.0
 h2==4.1.0
+h5py==3.9.0
 hpack==4.0.0
 httpcore==0.17.3
+httplib2==0.22.0
 httpx==0.24.1
+huggingface-hub==0.16.4
 hyperframe==6.0.1
 idna==3.4
+importlib-metadata==6.8.0
+Jinja2==3.1.2
+joblib==1.3.2
+keras==2.13.1
 kombu==5.3.1
 langchain==0.0.247
+langcodes==3.3.0
 langsmith==0.0.15
+libclang==16.0.6
+llama-cpp-python==0.1.78
 loguru==0.7.0
 lxml==4.9.3
+Markdown==3.4.4
+MarkupSafe==2.1.3
 marshmallow==3.20.1
+mpmath==1.3.0
 multidict==6.0.4
+murmurhash==1.0.9
 mypy-extensions==1.0.0
+networkx==3.1
+nltk==3.8.1
 numexpr==2.8.4
-numpy==1.25.1
+numpy==1.24.3
+oauth2client==4.1.3
+oauthlib==3.2.2
 openai==0.27.8
 openapi-schema-pydantic==1.2.4
+opt-einsum==3.3.0
 packaging==23.1
+pathy==0.10.2
 Pillow==10.0.0
 pinecone-client==2.2.2
 portalocker==2.7.0
+preshed==3.0.8
 prompt-toolkit==3.0.39
-protobuf==4.23.4
+protobuf==3.20.3
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
 pycparser==2.21
 pydantic==1.10.12
 PyMySQL==1.1.0
+pyparsing==3.1.1
 pypdf==3.14.0
 pypdfium2==4.18.0
 python-dateutil==2.8.2
@@ -61,18 +105,50 @@ qdrant-client==1.3.2
 redis==4.6.0
 regex==2023.6.3
 requests==2.31.0
+requests-oauthlib==1.3.1
+rsa==4.9
+safetensors==0.3.3
+scikit-learn==1.3.0
+scipy==1.11.2
+sentence-transformers==2.2.2
+sentencepiece==0.1.99
 six==1.16.0
+smart-open==6.3.0
 sniffio==1.3.0
 soupsieve==2.4.1
+spacy==3.6.1
+spacy-legacy==3.0.12
+spacy-loggers==1.0.4
 SQLAlchemy==2.0.19
 sqlparse==0.4.4
+srsly==2.4.7
+sympy==1.12
 tenacity==8.2.2
+tensorboard==2.13.0
+tensorboard-data-server==0.7.1
+tensorflow==2.13.0
+tensorflow-estimator==2.13.0
+tensorflow-hub==0.14.0
+tensorflow-macos==2.13.0
+termcolor==2.3.0
+thinc==8.1.12
+threadpoolctl==3.2.0
 tiktoken==0.4.0
+tokenizers==0.13.3
+torch==2.0.1
+torchvision==0.15.2
 tqdm==4.65.0
+transformers==4.32.0
+typer==0.9.0
 typing-inspect==0.9.0
-typing_extensions==4.7.1
+typing_extensions==4.5.0
 tzdata==2023.3
+uritemplate==3.0.1
 urllib3==1.26.16
 vine==5.0.0
+wasabi==1.1.2
 wcwidth==0.2.6
+Werkzeug==2.3.7
+wrapt==1.15.0
 yarl==1.9.2
+zipp==3.16.2