Adding support for qdrant and bumping langchain version

common.env

@@ -1,4 +1,18 @@
+# Required for openai azure ------------------------------------------------------------------
+USE_AZURE_OPENAI=true
+AZURE_OPENAI_API_KEY=5c25fe261edc4b0eb10417e5cda51689
+AZURE_OPENAI_API_INSTANCE_NAME=shanurrahman
+AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME=emailclassifier
+AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=emailclassifierembedding
+AZURE_OPENAI_API_VERSION=2023-03-15-preview
+# --------------------------------------------------------------------------------------------
+
 OPENAI_API_KEY=
-PINECONE_API_KEY=
-PINECONE_ENVIRONMENT=
-PINECONE_INDEX_NAME=
+PINECONE_API_KEY=2da8059e-748e-4e15-9f14-aa6f1d21a778
+PINECONE_ENVIRONMENT=asia-southeast1-gcp-free
+PINECONE_INDEX_NAME=kurtzweil
+
+
+# QDRANT_URL
+QDRANT_URL=http://qdrant:6333
+STORE=qdrant

docker-compose.yml

@@ -50,6 +50,15 @@ services:
     networks:
       - openchat_network
 
+  qdrant:
+    image: qdrant/qdrant
+    ports:
+      - 6333:6333
+    volumes:
+      - ./qdrant_storage:/qdrant/storage
+    networks:
+      - openchat_network
+
   redis:
     restart: unless-stopped
     image: "redis:3.2"

llm-server/config/pinecone.ts

@@ -10,4 +10,6 @@ const PINECONE_INDEX_NAME = process.env.PINECONE_INDEX_NAME ?? '';
 
 const PINECONE_NAME_SPACE = 'bot-test'; //namespace is optional for your vectors
 
+
+export const PINECONE_TEXT_KEY = 'text';
 export {PINECONE_INDEX_NAME, PINECONE_NAME_SPACE};

llm-server/data-sources/codebaseHandler.ts

@@ -2,9 +2,9 @@ import type {NextApiRequest, NextApiResponse} from 'next';
 import {PINECONE_INDEX_NAME} from '@/config/pinecone';
 import {RecursiveCharacterTextSplitter} from 'langchain/text_splitter';
 import {OpenAIEmbeddings} from 'langchain/embeddings/openai';
-import {PineconeStore} from 'langchain/vectorstores/pinecone';
 import {pinecone} from '@/utils/pinecone-client';
 import {GithubRepoLoader} from "langchain/document_loaders/web/github";
+import { initVectorStore } from '@/utils/initVectorStore';
 
 export default async function codebaseHandler(req: NextApiRequest, res: NextApiResponse) {
     try {
@@ -28,9 +28,7 @@ export default async function codebaseHandler(req: NextApiRequest, res: NextApiR
         const embeddings = new OpenAIEmbeddings();
         const index = pinecone.Index(PINECONE_INDEX_NAME);
 
-        await PineconeStore.fromDocuments(docs, embeddings, {
-            pineconeIndex: index, namespace: namespace, textKey: 'text',
-        });
+        await initVectorStore(docs, embeddings, {index, namespace})
 
         console.log('Indexed documents. all done!')
         return res.status(200).json({message: 'Success'});

llm-server/data-sources/pdfHandler.ts

@@ -1,11 +1,11 @@
-import type {NextApiRequest, NextApiResponse} from 'next';
-import {PINECONE_INDEX_NAME} from '@/config/pinecone';
-import {DirectoryLoader} from 'langchain/document_loaders/fs/directory';
-import {CustomPDFLoader} from '@/utils/customPDFLoader';
-import {RecursiveCharacterTextSplitter} from 'langchain/text_splitter';
-import {OpenAIEmbeddings} from 'langchain/embeddings/openai';
-import {PineconeStore} from 'langchain/vectorstores/pinecone';
-import {pinecone} from '@/utils/pinecone-client';
+import { PINECONE_INDEX_NAME } from '@/config/pinecone';
+import { CustomPDFLoader } from '@/utils/customPDFLoader';
+import { initVectorStore } from '@/utils/initVectorStore';
+import { pinecone } from '@/utils/pinecone-client';
+import { DirectoryLoader } from 'langchain/document_loaders/fs/directory';
+import { OpenAIEmbeddings } from 'langchain/embeddings/openai';
+import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
+import type { NextApiRequest, NextApiResponse } from 'next';
 
 export default async function pdfHandler(req: NextApiRequest, res: NextApiResponse) {
     try {
@@ -27,9 +27,7 @@ export default async function pdfHandler(req: NextApiRequest, res: NextApiRespon
         const embeddings = new OpenAIEmbeddings();
         const index = pinecone.Index(PINECONE_INDEX_NAME);
 
-        await PineconeStore.fromDocuments(docs, embeddings, {
-            pineconeIndex: index, namespace: namespace, textKey: 'text',
-        });
+        await initVectorStore(docs, embeddings, {index, namespace});
 
         console.log('All is done, folder deleted');
         return res.status(200).json({message: 'Success'});

llm-server/data-sources/websiteHandler.ts

@@ -4,8 +4,8 @@ import {DirectoryLoader} from 'langchain/document_loaders/fs/directory';
 import {RecursiveCharacterTextSplitter} from 'langchain/text_splitter';
 import {TextLoader} from 'langchain/document_loaders';
 import {OpenAIEmbeddings} from 'langchain/embeddings/openai';
-import {PineconeStore} from 'langchain/vectorstores/pinecone';
 import {pinecone} from '@/utils/pinecone-client';
+import { initVectorStore } from '@/utils/initVectorStore';
 
 export default async function websiteHandler(req: NextApiRequest, res: NextApiResponse) {
     try {
@@ -27,9 +27,7 @@ export default async function websiteHandler(req: NextApiRequest, res: NextApiRe
         const embeddings = new OpenAIEmbeddings();
         const index = pinecone.Index(PINECONE_INDEX_NAME);
 
-        await PineconeStore.fromDocuments(docs, embeddings, {
-            pineconeIndex: index, namespace: namespace, textKey: 'text',
-        });
+        await initVectorStore(docs, embeddings, {index, namespace})
         console.log('All is done, folder deleted');
         return res.status(200).json({message: 'Success'});
     } catch (e) {

llm-server/interfaces/storeOptions.interface.ts

@@ -0,0 +1,6 @@
+import { VectorOperationsApi } from "@pinecone-database/pinecone/dist/pinecone-generated-ts-fetch";
+
+export interface StoreOptions {
+    namespace?: string;
+    index?: VectorOperationsApi;
+  }