Cleaned the code, better structure

.gitignore

@@ -5,3 +5,5 @@ t.py
 rag/
 DockerFile
 s.txt
+__pycache__
+

README.md

@@ -43,9 +43,9 @@ py setup.py install
 ```
 4. Set up the environment variables
 ```bash
-export GROQ_API_KEY="your_groq_api_key"
-export COHERE_API_KEY="your_cohere_api_key"
-export HUGGINGFACE_API_KEY="your_hugging
+export GROQ_API_KEY="your_groq_key"
+export COHERE_API_KEY="your_cohere_key"
+export HUGGINGFACE_API_KEY="your_huggingFace_key"
 ```
 
 5. Run the `app.py` file

app.py

@@ -1,18 +1,4 @@
-import src.data_processing.get_embeddings
-from data.process_data import load_documents, embed_and_store_documents, split_documents
-from langchain.prompts import ChatPromptTemplate
-from groq import Groq
-import os
-from src.database.chroma_search_functions import get_relevant_data
-
-"""
-    Importing the functions and setting up the environment variables
-
-"""
-
-client = Groq(
-    api_key=os.getenv("GROQ_API_KEY"),
-)
+from src.main_reasoning import reasoning
 
 
 """
@@ -44,51 +30,16 @@
 """
 
 
-
-
-def format_context(context):
-    return "\n\n".join([f"Chunk {i+1}: {chunk}" for i, chunk in enumerate(context)])
-
-def check_and_process_documents():
-    path = "data/processed/chroma"
-    print(f"Checking if path exists: {path}")
-
-    if not os.path.exists(path):
-        print(f"Path does not exist: {path}")
-
-        documents = load_documents()
-        print("Documents loaded")
-
-        chunks = split_documents(documents)
-        print("Documents split into chunks")
-
-        embed_and_store_documents(chunks)
-        print("Documents embedded and stored")
-    else:
-        print(f"Path already exists: {path}")
-
 def main():
-
-    """
-    loading documents should be performed only once, ti will take a bit of time at first.
-    You can comment them out as chromaDB has the infos already
 
-    """
-    check_and_process_documents()
 
-    if not os.path.exists("data/processed/chroma"):
-        documents = load_documents()
-        chunks = split_documents(documents)
-        embed_and_store_documents(chunks)
-        print("Documents loaded, split, and stored")
-
 
+    query = "How to enter prison in Monopoly?"
 
 
-    query = "How to enter prison in Monopoly?"
 
     PROMPT_TEMPLATE = """
-    Answer this question in a clear, unboring matter,  based on the follwing context:
+    Answer this question in a clear, unboring matter, based on the follwing context:
     {context}
 
     -----
@@ -99,66 +50,7 @@ def main():
     Answer:
     """
 
-    results = get_relevant_data(query)
-
-    # context_text= "\n\n---\n\n".join([doc.page_content for doc, _score in results])
-    prompt_template = ChatPromptTemplate.from_template(PROMPT_TEMPLATE)
-    prompt = prompt_template.format(context=results, question=query)
-
-    print("#"*100 + "\n\n")
-
-
-    chat_completion = client.chat.completions.create(
-        #
-        # Required parameters
-        #
-        messages=[
-                # Set an optional system message. This sets the behavior of the
-                # assistant and can be used to provide specific instructions for
-                # how it should behave throughout the conversation.
-                {
-                    "role": "system",
-                    "content": prompt
-                },
-                # Set a user message for the assistant to respond to.
-                {
-                    "role": "user",
-                    "content": query,
-                }
-            ],
-
-            # The language model which will generate the completion.
-            model="llama3-70b-8192",
-
-            #
-            # Optional parameters
-            #
-
-            # Controls randomness: lowering results in less random completions.
-            # As the temperature approaches zero, the model will become deterministic
-            # and repetitive.
-            temperature=0.5,
-
-            # The maximum number of tokens to generate. Requests can use up to
-            # 2048 tokens shared between prompt and completion.
-            max_tokens=1024,
-
-            # Controls diversity via nucleus sampling: 0.5 means half of all
-            # likelihood-weighted options are considered.
-            top_p=1,
-
-            # A stop sequence is a predefined or user-specified text string that
-            # signals an AI to stop generating content, ensuring its responses
-            # remain focused and concise. Examples include punctuation marks and
-            # markers like "[end]".
-            stop=None,
-
-            # If set, partial message deltas will be sent.
-            stream=False,
-        )
-
-
-    response = chat_completion.choices[0].message.content
+    response = reasoning(query, PROMPT_TEMPLATE)
 
     print("Response: ", response)
 

data/process_data.py

@@ -2,19 +2,7 @@
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.document_loaders import PyPDFDirectoryLoader
 from langchain.schema.document import Document
-import os
 from src.data_processing.get_embeddings import get_embeddings
-from dotenv import load_dotenv
-
-
-load_dotenv()
-
-"""
-    Initializating the APIs and setting up the environment variables
-
-"""
-
-api_key = os.getenv("COHERE_API_KEY")
 
 CHROMA_PATH = "data/processed/chroma"
 DATA_PATH = "data"

src/database/chroma_search_functions.py

@@ -1,31 +1,15 @@
-from langchain_community.vectorstores import Chroma
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_community.document_loaders import PyPDFDirectoryLoader
-from langchain.schema.document import Document
+# from langchain.text_splitter import RecursiveCharacterTextSplitter
+# from langchain_community.document_loaders import PyPDFDirectoryLoader
+# from langchain.schema.document import Document
 # from FlagEmbedding.flag_models import FlagModel
 # from FlagEmbedding.flag_reranker import FlagReranker
+from langchain_community.vectorstores import Chroma
 from src.data_processing.get_embeddings import get_embeddings
-import uuid
-import os
-from dotenv import load_dotenv
-import cohere
-
-load_dotenv()
-
-"""
-    Initializating the APIs and setting up the environment variables
-
-"""
+from src.models.models import cohere_reranker
 
-api_key = os.getenv("COHERE_API_KEY")
 
 CHROMA_PATH = "data/processed/chroma"
 
-# init client
-co = cohere.Client(api_key)
-
-
-
 
 # load the data
 def get_chroma_db(get_embeddings=get_embeddings):
@@ -67,6 +51,7 @@ def format_context(context):
     return "\n\n".join([f"Chunk {i+1}: {chunk}" for i, chunk in enumerate(context)])
 
 
+
 def reranked_documents(query, long_string, top_k=3):
     # Split the long string into individual chunks using '\n\n---\n\n' as the separator
     chunks = long_string.split("\n\n---\n\n")
@@ -77,23 +62,20 @@ def reranked_documents(query, long_string, top_k=3):
     if not valid_chunks:
         print("No valid chunks to rerank.")
         return []
-
-    # Use the cohere rerank API
-    rerank_docs = co.rerank(
-        query=query,
-        documents=valid_chunks,
-        top_n=top_k,
-        model="rerank-english-v2.0"
-    )
+
+    # cohere reranker
+    rerank_docs = cohere_reranker(query, valid_chunks, top_k)
 
     print("#"*100 + "\n\n")
+
     # Extract and print reranked chunks using the indices from the rerank response
     reranked_chunks = [valid_chunks[result.index] for result in rerank_docs.results]
     print("Reranked Chunks:\n\n", format_context(reranked_chunks))
 
     return reranked_chunks
 
 
+
 def get_relevant_data(query):
     retrieved_chunks = retrieve_documents(query)
     reranked_chunks = reranked_documents(query, retrieved_chunks)

src/main_reasoning.py

@@ -0,0 +1,43 @@
+from data.process_data import load_documents, embed_and_store_documents, split_documents
+from langchain.prompts import ChatPromptTemplate
+from src.database.chroma_search_functions import get_relevant_data
+from src.models.models import llama_groq
+import os
+
+
+def format_context(context):
+    return "\n\n".join([f"Chunk {i+1}: {chunk}" for i, chunk in enumerate(context)])
+
+
+def check_and_process_documents():
+    path = "data/processed/chroma"
+    print(f"Checking if path exists: {path}")
+
+    if not os.path.exists(path):
+        print(f"Path does not exist: {path}")
+
+        documents = load_documents()
+        print("Documents loaded")
+
+        chunks = split_documents(documents)
+        print("Documents split into chunks")
+
+        embed_and_store_documents(chunks)
+        print("Documents embedded and stored")
+    else:
+        print(f"Path already exists: {path}")
+
+
+
+def reasoning(query, prompt):
+
+    check_and_process_documents()
+
+    print("#"*100 + "\n\n")
+
+    results = get_relevant_data(query)
+
+    prompt_template = ChatPromptTemplate.from_template(prompt)
+    prompt = prompt_template.format(context=results, question=query)
+    response = llama_groq(query, prompt)
+    return response

src/models/models.py

@@ -0,0 +1,81 @@
+from dotenv import load_dotenv
+from groq import Groq
+import cohere
+import os
+
+
+load_dotenv()
+
+
+client = Groq(
+    api_key=os.getenv("GROQ_API_KEY"),
+)
+
+# init client
+co = cohere.Client(os.getenv("COHERE_API_KEY"))
+
+
+def llama_groq(query, prompt):
+    chat_completion = client.chat.completions.create(
+        #
+        # Required parameters
+        #
+        messages=[
+                # Set an optional system message. This sets the behavior of the
+                # assistant and can be used to provide specific instructions for
+                # how it should behave throughout the conversation.
+                {
+                    "role": "system",
+                    "content": prompt
+                },
+                # Set a user message for the assistant to respond to.
+                {
+                    "role": "user",
+                    "content": query,
+                }
+            ],
+
+            # The language model which will generate the completion.
+            model="llama3-70b-8192",
+
+            #
+            # Optional parameters
+            #
+
+            # Controls randomness: lowering results in less random completions.
+            # As the temperature approaches zero, the model will become deterministic
+            # and repetitive.
+            temperature=0.5,
+
+            # The maximum number of tokens to generate. Requests can use up to
+            # 2048 tokens shared between prompt and completion.
+            max_tokens=1024,
+
+            # Controls diversity via nucleus sampling: 0.5 means half of all
+            # likelihood-weighted options are considered.
+            top_p=1,
+
+            # A stop sequence is a predefined or user-specified text string that
+            # signals an AI to stop generating content, ensuring its responses
+            # remain focused and concise. Examples include punctuation marks and
+            # markers like "[end]".
+            stop=None,
+
+            # If set, partial message deltas will be sent.
+            stream=False,
+        )
+
+
+    response = chat_completion.choices[0].message.content
+    return response
+
+
+def cohere_reranker(query, valid_chunks, top_k=3):
+    # Use the cohere rerank API
+    rerank_docs = co.rerank(
+        query=query,
+        documents=valid_chunks,
+        top_n=top_k,
+        model="rerank-english-v2.0"
+    )
+    return rerank_docs