The metadata is displayed in the chat. v1

dj_backend_server/.vscode/settings.json

@@ -2,5 +2,6 @@
     "[python]": {
         "editor.defaultFormatter": "ms-python.black-formatter"
     },
-    "python.formatting.provider": "none"
+    "python.formatting.provider": "black",
+    "editor.formatOnSave": false
 }

dj_backend_server/api/views/views_chat.py

@@ -43,33 +43,31 @@ def chat(request):
     """
     try:
 
-        logger.debug("Received chat request from view_messages.py - /api/chat/")
         body = json.loads(request.body.decode("utf-8"))
         question = body.get("question")
         namespace = body.get("namespace")
         mode = body.get("mode")
         initial_prompt = body.get("initial_prompt")
         token = body.get("token")
         session_id = body.get("session_id")
+        metadata = body.get("metadata", {})
 
         logger.debug(f"Request body parsed: {body}")
-        logger.debug(f"Question: {question}")
         bot = get_object_or_404(Chatbot, token=token)
-        logger.debug(f"Chatbot found: {bot.name}")
         if not question:
             return JsonResponse({"error": "No question in the request"}, status=400)
         sanitized_question = question.strip().replace("\n", " ")
-        logger.debug(f"Sanitized question: {sanitized_question}")
         vector_store = get_vector_store(StoreOptions(namespace=namespace))
-        logger.debug(f"Vector store obtained")
-        response_text = get_completion_response(
+
+        response_text, metadata = get_completion_response(
             vector_store=vector_store,
             initial_prompt=initial_prompt,
             mode=mode,
             sanitized_question=sanitized_question,
             session_id=session_id,
+            metadata=metadata,
         )
-        logger.debug(f"Response text: {response_text}")
+
         if isinstance(response_text, dict) and "text" in response_text:
             ChatHistory.objects.bulk_create(
                 [
@@ -90,9 +88,9 @@ def chat(request):
                 ]
             )
             logger.debug(
-                f"Response after creating ChatHistory: {json.dumps(response_text, indent=2)}"
+                f"Response after creating ChatHistory: {json.dumps(response_text, indent=2)}, metadata: {metadata}"
             )
-            return JsonResponse({"text": response_text})
+            return JsonResponse({"text": response_text, "metadata": metadata})
 
         elif isinstance(response_text, str):
             ChatHistory.objects.bulk_create(
@@ -114,9 +112,9 @@ def chat(request):
                 ]
             )
             logger.debug(
-                f"Response after creating ChatHistory 2: {json.dumps(response_text, indent=2)}"
+                f"Response after creating ChatHistory 2: {json.dumps(response_text, indent=2)}, metadata: {metadata}"
             )
-            return JsonResponse({"text": response_text})
+            return JsonResponse({"text": response_text, "metadata": metadata})
 
         else:
             return JsonResponse({"error": "Unexpected response from API"}, status=500)
@@ -132,7 +130,7 @@ def chat(request):
 
 
 def get_completion_response(
-    vector_store, mode, initial_prompt, sanitized_question, session_id
+    vector_store, mode, initial_prompt, sanitized_question, session_id, metadata
 ):
     """
     This function generates a response based on a given question. It uses either the 'retrieval_qa' or 'conversation_retrieval'
@@ -151,15 +149,18 @@ def get_completion_response(
         is a string, it is returned after removing markdown code block formatting.
     """
 
-    logger.debug(f"Entering get_completion_response function")
-    logger.debug(
-        f"Mode: {mode}, Initial Prompt: {initial_prompt}, Sanitized Question: {sanitized_question}, Session ID: {session_id}"
-    )
+    # logger.debug(f"Entering get_completion_response function")
+    # logger.debug(
+    #     f"Mode: {mode}, Initial Prompt: {initial_prompt}, Sanitized Question: {sanitized_question}, Session ID: {session_id}"
+    # )
     chain_type = os.getenv("CHAIN_TYPE", "conversation_retrieval")
     chain: QAWithSourcesChain
     if chain_type == "retrieval_qa":
         chain = getRetrievalQAWithSourcesChain(vector_store, mode, initial_prompt)
-        response = chain({"question": sanitized_question}, return_only_outputs=True)
+        response = chain.invoke(
+            {"question": sanitized_question, "metadata": metadata},
+            return_only_outputs=True,
+        )
         response_text = response["answer"]
         logger.debug(f"RetrievalQA response: {response_text}")
     elif chain_type == "conversation_retrieval":
@@ -171,13 +172,33 @@ def get_completion_response(
         logger.debug(f"Formatted Chat_history {chat_history}")
 
         response = chain.invoke(
-            {"question": sanitized_question, "chat_history": chat_history},
+            {
+                "question": sanitized_question,
+                "chat_history": chat_history,
+                "metadata": metadata,
+            },
         )
-        response_text = response.get("answer")
+        # Assuming 'response' is the JSON object you've provided
+        source_documents = response["source_documents"]
+
+        # Initialize an empty list to hold metadata from all documents
+        all_metadata = []
+
+        # Iterate through each document in the source documents
+        for document in source_documents:
+            # Correctly access the metadata attribute or method of the Document object
+            # Assuming the Document object has a 'metadata' attribute
+            metadata = document.metadata
+
+            # Add the metadata dictionary to the list
+            all_metadata.append(metadata)
+
+        response_text = response.get("answer", "")
+
     try:
         # Attempt to parse the response_text as JSON
         response_text = json.loads(response_text)
-        logger.debug(f"Response text after JSON parsing: {response_text}")
+
     except json.JSONDecodeError:
         # If response_text is not a JSON string, leave it as is
         pass
@@ -194,4 +215,5 @@ def get_completion_response(
             response_text.replace("```", "").replace("markdown\n", "").strip()
         )
         logger.debug(f"Response text after markdown removal: {response_text}")
-    return response_text
+    # print(f"metadata {metadata}")
+    return response_text, all_metadata

dj_backend_server/api/views/views_message.py

@@ -182,13 +182,9 @@ def send_chat(request):
         )  # {'from': 'user', 'type': 'text', 'content': 'input text from chat'}
         # Validate the request data
         content = data.get("content")
-        history = data.get("history")
-        logger.debug(f"Content: {content}")
-        logger.debug(
-            f"History: {history}"
-        )  # history is a list of chat history - None????
-        content_type = data.get("type")
-        metadata = data.get("metadata") or {}
+        # history = data.get("history")
+        # logger.debug(f"Content: {content}")
+        # logger.debug(f"History: {history}")
 
         session_id = get_session_id(request=request, bot_id=bot.id)
         history = ChatHistory.objects.filter(session_id=session_id)
@@ -219,11 +215,9 @@ def send_chat(request):
                 "history": history_entries,
                 "token": bot_token,
                 "session_id": session_id,
-                "metadata": metadata,
             },
             timeout=200,
         )
-        logger.debug(f"External API response: {response.text} and {response}")
 
         """
         This block will first check if the response content is not empty. If it is empty, 
@@ -242,7 +236,7 @@ def send_chat(request):
         else:
             try:
                 response_json = response.json()
-                logger.debug(f"Response JSON: {response_json}")
+                logger.debug(f"External API response 2")
             except json.JSONDecodeError:
                 logger.error("JSONDecodeError occurred")
                 return JsonResponse(
@@ -255,18 +249,45 @@ def send_chat(request):
                 )
 
         bot_response = ChatbotResponse(response.json())
-        # context = {'APP_URL': settings.APP_URL, session_id: session_id}
+        for metadata_entry in response_json.get("metadata", []):
+            print(f"Last Update: {metadata_entry.get('last_update')}")
+            print(f"Type: {metadata_entry.get('type')}")
+            print(f"Original Filename: {metadata_entry.get('original_filename')}")
+            print(f"Source: {metadata_entry.get('source')}")
+            print(f"Folder: {metadata_entry.get('folder')}")
+            print(f"Page: {metadata_entry.get('page')}")
+            print(f"Bot ID: {metadata_entry.get('bot_id')}")
+            print(f"ID: {metadata_entry.get('_id')}")
+            print(f"Collection Name: {metadata_entry.get('_collection_name')}")
+
+        metadata_items = [
+            {
+                "source": entry.get("source"),
+                "original_filename": entry.get("original_filename"),
+            }
+            for entry in response_json.get("metadata", [])
+        ]
+        metadata_html = render_to_string(
+            "widgets/metadata.html",
+            {
+                "APP_URL": settings.APP_URL,
+                "session_id": session_id,
+                "metadata_items": metadata_items,
+            },
+        )
         feedback_form_html = render_to_string(
             "widgets/feedback.html",
             {"APP_URL": settings.APP_URL, "session_id": session_id},
         )
+
         print(f"Response in JSON {session_id}")
+        html_compose = metadata_html + feedback_form_html
         return JsonResponse(
             {
                 "type": "text",
                 "response": {
                     "text": bot_response.get_bot_reply(),
-                    "html": feedback_form_html,
+                    "html": html_compose,
                     "session_id": session_id,
                 },
             }

dj_backend_server/web/services/chat_history_service.py

@@ -54,6 +54,6 @@ def get_chat_history_for_retrieval_chain(
                 memory.save_context({"input": user_query}, {"output": entry.message})
                 user_query = None
 
-    logger.debug(f"Memory PRINT: {memory}")
+    # logger.debug(f"Memory PRINT: {memory}")
     # chat_history = memory.load_memory_variables({})
     return chat_history

dj_backend_server/web/templates/widgets/metadata.html

@@ -0,0 +1,16 @@
+{% load i18n %}
+{% load static %}
+
+<div class="my-5">
+    <div class="w-full block h-px bg-gray-200 my-2"></div>
+    <div style="color: #000; background-color: #c3c3c3; transition: all 0.2s; padding: 0.25rem 0.5rem; border-radius: 0.375rem;">
+        <div>
+            {% for item in metadata_items %}
+                <div style="margin-bottom: 10px;">
+                    <strong>{% trans "Source:" %}</strong> {{ item.source }}<br>
+                    <strong>{% trans "Original Filename:" %}</strong> {{ item.original_filename }}
+                </div>
+            {% endfor %}
+        </div>
+    </div>
+</div>