RAG_Institutional_Database.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# env lib imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install pdfminer.six\n",
    "# pip install -U langchain-community\n",
    "# !pip install pypdf\n",
    "# pip install chromadb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RAG\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import transformers\n",
    "# from pathlib import Path\n",
    "# from xml.dom.minidom import Document\n",
    "# import langchain_community, langchain\n",
    "# from langchain.vectorstores import Chroma\n",
    "# from langchain.prompts import ChatPromptTemplate\n",
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
    "\n",
    "import uuid\n",
    "import chromadb\n",
    "from json import load\n",
    "from llama_index.readers.file import PyMuPDFReader\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "# provide path to a JSON with the API key\n",
    "from huggingface_hub import login\n",
    "HUGGING_FACE_API_KEY = load(open('../../config.json'))['HUGGING_FACE_API_KEY']\n",
    "login(token=HUGGING_FACE_API_KEY)\n",
    "\n",
    "import json\n",
    "with open('../../config.json') as config_file: config = json.load(config_file)\n",
    "KEY_TOGETHERAI = config['KEY_TOGETHERAI']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Paths\n",
    "Have a folder named file_dump to dump all the pdf of the institute "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "CHROMA_PATH = \"institute\"  # Name of your vector database\n",
    "DUMP_FOLD_PATH = \"data/file_dump\"  # Name of your dump folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(DUMP_FOLD_PATH):\n",
    "    os.makedirs(DUMP_FOLD_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import urljoin\n",
    "\n",
    "# Function to download a PDF\n",
    "def download_pdf(pdf_url, save_path):\n",
    "    response = requests.get(pdf_url)\n",
    "    if response.status_code == 200:\n",
    "        with open(save_path, 'wb') as file:\n",
    "            file.write(response.content)\n",
    "            print(f\"Downloaded: {save_path}\")\n",
    "    else:\n",
    "        print(f\"Failed to download: {pdf_url}\")\n",
    "\n",
    "# Main scraping function\n",
    "def scrape_pdfs_from_website(base_url, output_folder):\n",
    "    # Make HTTP request to the base URL\n",
    "    response = requests.get(base_url)\n",
    "    if response.status_code != 200:\n",
    "        print(f\"Failed to access {base_url}\")\n",
    "        return\n",
    "    \n",
    "    # Parse the HTML content\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "\n",
    "    # Find all anchor tags with PDF links\n",
    "    pdf_links = []\n",
    "    for link in soup.find_all('a', href=True):\n",
    "        href = link['href']\n",
    "        if href.endswith('.pdf'):\n",
    "            pdf_links.append(urljoin(base_url, href))  # Convert relative links to absolute\n",
    "    \n",
    "    # Ensure the output folder exists\n",
    "    os.makedirs(output_folder, exist_ok=True)\n",
    "    \n",
    "    # Download all PDF files\n",
    "    for pdf_url in pdf_links:\n",
    "        pdf_name = os.path.basename(pdf_url)\n",
    "        save_path = os.path.join(output_folder, pdf_name)\n",
    "        download_pdf(pdf_url, save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloaded: data/file_dump/Final Placement Brochure 2024-25.pdf\n",
      "Downloaded: data/file_dump/BTech-Ordinances.pdf\n",
      "Downloaded: data/file_dump/2024-May-UG Regulations.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-UG%20Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-August-UG-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-July-UG-Regulations.pdf\n",
      "Downloaded: data/file_dump/2017-July-UG-Regulations.pdf\n",
      "Downloaded: data/file_dump/2024-May-BTech(CSE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-BTech(CSE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019%20-August-BTech(CSE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-July-BTech(CSE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2017%20-July-BTech(CSE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2016%20-BTech(CSE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2024-May-BTech(ECE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-BTech(ECE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019%20-August-BTech(ECE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-July-BTech(ECE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2017%20-July-BTech(ECE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2016%20-BTech(ECE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2024-May-BTech(CSAM)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-BTech(CSAM)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-%20August%20-BTech(CSAM)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-July-BTech(CSAM)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2017-%20July-BTech(CSAM)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2017-April-BTech(CSAM)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2024-May-BTech(CSAI)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-BTech(CSAI)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-%20August%20-BTech(CSAI)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2024-May-BTech(CSD)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-BTech(CSD)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-August-B.Tech(CSD)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-July-BTech(CSD)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2017-July-B.Tech(CSD)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2024-May-BTech(CSSS)-Regulations.pdf\n",
      "Downloaded: data/file_dump/Appendix%20II_CSSS%20Regulations%202022-13jan23.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-BTech(CSSS)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2020-June-B.Tech(CSSS)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-July-BTech(CSSS)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2018-January-B.Tech.(CSSS)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2017-July-B.Tech(ITSS)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2024-August-BTech(CSB)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-BTech(CSB)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-August%20-B.Tech(CSB)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2019-July-BTech(CSB)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2018-July-B.Tech.%20(CSB)-Regulation.pdf\n",
      "Downloaded: data/file_dump/2024-May-BTech(EVE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/Specific-regulation%202022-July-BTech(EVE).pdf\n",
      "Downloaded: data/file_dump/2024-May-Minor in Economics.pdf\n",
      "Downloaded: data/file_dump/2018-July-Regulation-for-Minor-in-Economics.pdf\n",
      "Downloaded: data/file_dump/2014%20July-Minor%20in%20Economics.pdf\n",
      "Downloaded: data/file_dump/2024-May-Regulations for Minor in Computational Biology.pdf\n",
      "Downloaded: data/file_dump/2018%20July%20Regulation%20for%20Minor%20in%20Computational%20Biology%20(CB).pdf\n",
      "Downloaded: data/file_dump/2024-July-Regulations for Minor in Entrepreneurship.pdf\n",
      "Downloaded: data/file_dump/Entrepreneurship%20Minor%20Program_final_proposed_structure%20(1).pdf\n",
      "Downloaded: data/file_dump/2020-April-Minor%20in%20Entrepreneurship.pdf\n",
      "Downloaded: data/file_dump/2024-May-Minor in Quantum Technologies.pdf\n",
      "Downloaded: data/file_dump/Minor%20in%20Quantum%20Technologies%20Aug%202022.pdf\n",
      "Downloaded: data/file_dump/PG-Ordinances.pdf\n",
      "Downloaded: data/file_dump/2024-July-PG-Regulations_V1.12.pdf\n",
      "Downloaded: data/file_dump/2023-February-PG-Regulations.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-PG-Regulations_Final.pdf\n",
      "Downloaded: data/file_dump/2020-April-PG-Regulations_Final.pdf\n",
      "Downloaded: data/file_dump/2017-July-PG-Regulations.pdf\n",
      "Downloaded: data/file_dump/2024-August-M.Tech.( CSE)-Regulations1.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-M.Tech.(%20CSE)-Regulations_Final.pdf\n",
      "Downloaded: data/file_dump/2020-%20July-M.Tech.(%20CSE)-Regulations_Final.pdf\n",
      "Downloaded: data/file_dump/2017-July-MTech(CSE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2016%20July-MTech(CSE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2024-May-M.Tech.(CS & AI)-Regulation.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-M.Tech.(CS%20&%20AI)-Regulation_Final.pdf\n",
      "Downloaded: data/file_dump/2018-July-M.Tech.(CS%20&%20AI)-Regulation.pdf\n",
      "Downloaded: data/file_dump/Regulations%20MTech%20(R).pdf\n",
      "Downloaded: data/file_dump/2024-May-M.Tech.(ECE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2023-Mar-M.Tech.(ECE)-Regulations_Final.pdf\n",
      "Downloaded: data/file_dump/2021-Jan-M.Tech.(ECE)-Regulations_Final.pdf\n",
      "Downloaded: data/file_dump/2020-%20July-M.Tech.(%20ECE)-Regulations_Final.pdf\n",
      "Downloaded: data/file_dump/2017-%20July-M.Tech.(%20ECE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2016%20July-M.Tech.(%20ECE)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2024- July-M.Tech.(CB)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2021-%20Jan-M.Tech.(CB)-Regulations_Final.pdf\n",
      "Downloaded: data/file_dump/2018-January-M.Tech.%20CB_Regulations.pdf\n",
      "Downloaded: data/file_dump/2017-%20July-M.Tech(CB)-Regulations.pdf\n",
      "Downloaded: data/file_dump/2024 -July- Dual Degree Regulations CSE & ECE.pdf\n",
      "Downloaded: data/file_dump/2021%20January-%20Dual%20Degree%20Regulations_Final.pdf\n",
      "Downloaded: data/file_dump/2020%20April-%20Dual%20Degree%20Regulations_Final.pdf\n",
      "Downloaded: data/file_dump/2016-%20Dual%20Degree%20Regulations.pdf\n",
      "Downloaded: data/file_dump/2024-July-Dual Degree Regulations CB.pdf\n",
      "Downloaded: data/file_dump/Regulations%20of%20MTech%20Dual%20Degree%20in%20CB.pdf\n",
      "Downloaded: data/file_dump/PDA for Phd Students10072023.pdf\n",
      "Downloaded: data/file_dump/Internship Guidelines for M.Tech. (CSE) 2024 and Future Batches.pdf\n",
      "Downloaded: data/file_dump/Internship Guidelines for M.Tech. 2020 and Future Batches.pdf\n",
      "Downloaded: data/file_dump/Guidelines for Dean_Academic Awards 05092024.pdf\n",
      "Downloaded: data/file_dump/Penalty for Submission of any Forged Documents or Misbehaviour by students in any form.pdf\n",
      "Downloaded: data/file_dump/Travel_Financial Assistance for M.Tech (Research) CSE.pdf\n",
      "Downloaded: data/file_dump/TA Performance, Attendance & Policy_UG & PG.pdf\n",
      "Downloaded: data/file_dump/Guidelines for Various Awards in Convocation.pdf\n",
      "Downloaded: data/file_dump/CW  Guidelines_Updated_SA_28.12.20.pdf\n",
      "Downloaded: data/file_dump/Self Growth Guidelines.pdf\n",
      "Downloaded: data/file_dump/Procedure for using SG_CW_Portal.pdf\n",
      "Downloaded: data/file_dump/Office Order - Travel Assistance for Btech & Mtech Students (Revised).pdf\n",
      "Downloaded: data/file_dump/THESIS-SP-CP-II Evaluation Process(Student Copy) -10112020_Final-9june22.pdf\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import urljoin\n",
    "\n",
    "# Function to scrape text from a page and save it\n",
    "def scrape_page_text(page_url, output_folder, file_name):\n",
    "    try:\n",
    "        response = requests.get(page_url)\n",
    "        if response.status_code != 200:\n",
    "            print(f\"Failed to access {page_url}\")\n",
    "            return\n",
    "\n",
    "        # Parse the HTML content\n",
    "        soup = BeautifulSoup(response.text, 'html.parser')\n",
    "        \n",
    "        # Extract all text\n",
    "        page_text = soup.get_text(separator='\\n').strip()\n",
    "        \n",
    "        # Save text to a file\n",
    "        os.makedirs(output_folder, exist_ok=True)\n",
    "        file_path = os.path.join(output_folder, f\"{file_name}.txt\")\n",
    "        with open(file_path, 'w', encoding='utf-8') as file:\n",
    "            file.write(page_text)\n",
    "            print(f\"Saved: {file_path}\")\n",
    "    except Exception as e:\n",
    "        print(f\"Error scraping {page_url}: {e}\")\n",
    "\n",
    "# Function to find all links on the main page\n",
    "def scrape_website(base_url, output_folder):\n",
    "    try:\n",
    "        response = requests.get(base_url)\n",
    "        if response.status_code != 200:\n",
    "            print(f\"Failed to access {base_url}\")\n",
    "            return\n",
    "\n",
    "        # Parse the HTML content\n",
    "        soup = BeautifulSoup(response.text, 'html.parser')\n",
    "        \n",
    "        # Find all page links (anchor tags with href)\n",
    "        page_links = []\n",
    "        for link in soup.find_all('a', href=True):\n",
    "            href = link['href']\n",
    "            # Filter to avoid external links or specific conditions (if needed)\n",
    "            if href.startswith('/'):  # Adjust as per website structure\n",
    "                full_url = urljoin(base_url, href)\n",
    "                page_links.append(full_url)\n",
    "        \n",
    "        # Scrape each page and save text\n",
    "        for i, page_url in enumerate(page_links):\n",
    "            file_name = f\"page_{i+1}\"  # Custom file naming\n",
    "            scrape_page_text(page_url, output_folder, file_name)\n",
    "    except Exception as e:\n",
    "        print(f\"Error processing {base_url}: {e}\")\n",
    "\n",
    "# # Example Usage\n",
    "# base_url = \"https://example.com\"  # Replace with your target website\n",
    "# output_folder = \"website_text\"\n",
    "# scrape_website(base_url, output_folder)\n",
    "\n",
    "# Example Usage\n",
    "base_url = \"https://www.iiitd.ac.in/academics/resources\"\n",
    "output_folder = DUMP_FOLD_PATH\n",
    "scrape_pdfs_from_website(base_url, output_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def __load_documents(DUMP_FOLD_PATH):\n",
    "    pdf_paths = [f for f in os.listdir(DUMP_FOLD_PATH) if f.endswith('.pdf')]\n",
    "    txt_paths = [f for f in os.listdir(DUMP_FOLD_PATH) if f.endswith('.txt')]\n",
    "    documents = []  # List of documents\n",
    "    for pdf_path in pdf_paths: \n",
    "        chunks, doc_idxs = __extract_text_from_pdf(pdf_path, loader=\"PyMuPDFReader\")\n",
    "        documents.append(chunks)\n",
    "    for txt_path in txt_paths:\n",
    "        with open(os.path.join(DUMP_FOLD_PATH, txt_path), 'r') as f:\n",
    "            documents.append(f.read())\n",
    "    return documents\n",
    "def __process_documents(documents):\n",
    "    # documents is list of list of chunks i want a only list of chunks\n",
    "    documents = [chunk for doc in documents for chunk in doc]\n",
    "\n",
    "    # Chunck processing\n",
    "    documents = [chunk for chunk in documents if chunk != \"\"]\n",
    "    documents = [chunk.replace(\"\\n\", \"\") for chunk in documents]\n",
    "    return documents\n",
    "\n",
    "def __extract_text_from_pdf(file_path, folder_path=DUMP_FOLD_PATH, loader=None):\n",
    "    file_path = os.path.join(folder_path, file_path)\n",
    "\n",
    "    if loader == \"PyPDF2\":\n",
    "        loader = PyPDFLoader(file_path)\n",
    "        pages = loader.load()\n",
    "\n",
    "        # Split document into chunks (e.g., chunk_size=500)\n",
    "        text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)\n",
    "        chunks = text_splitter.split_documents(pages)\n",
    "        return chunks, None\n",
    "\n",
    "    elif loader == \"PyMuPDFReader\":\n",
    "        loader = PyMuPDFReader()\n",
    "        documents = loader.load(file_path=file_path)\n",
    "\n",
    "        # Text splitter to split the document\n",
    "        from llama_index.core.node_parser import SentenceSplitter\n",
    "        text_parser = SentenceSplitter(\n",
    "            chunk_size=1024,\n",
    "            # separator=\" \",\n",
    "        )\n",
    "\n",
    "        text_chunks = []\n",
    "        # maintain relationship with source doc index, to help inject doc metadata in (3)\n",
    "        doc_idxs = []\n",
    "        for doc_idx, doc in enumerate(documents):\n",
    "            cur_text_chunks = text_parser.split_text(doc.text)\n",
    "            text_chunks.extend(cur_text_chunks)\n",
    "            doc_idxs.extend([doc_idx] * len(cur_text_chunks))\n",
    "        \n",
    "        return (text_chunks, doc_idxs)\n",
    "    else:\n",
    "        raise ValueError(\"Invalid loader\")\n",
    "\n",
    "def __create_collection(collection_name=CHROMA_PATH):\n",
    "    chroma_client = chromadb.Client()\n",
    "    # collection = chroma_client.create_collection(name=CHROMA_PATH)\n",
    "    collection = chroma_client.get_or_create_collection(name=collection_name)\n",
    "    embedder = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "    return collection, embedder\n",
    "def __add_documents_to_collection(documents, rag_objects):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "        documents: list of documents (str)\n",
    "        rag_objects: tuple of (collection, embedder)\n",
    "        \n",
    "    Returns:\n",
    "        collection: collection of documents\n",
    "    \"\"\"\n",
    "    collection, embedder = rag_objects\n",
    "\n",
    "    # Dynamically retrieve the maximum batch size from the server\n",
    "    max_batch_size = collection._client.get_max_batch_size()\n",
    "\n",
    "    embeddings = embedder.encode(documents)\n",
    "    ids = [str(uuid.uuid4()) for _ in documents]  # Adjust this for actual document IDs/names\n",
    "\n",
    "    # Split data into batches\n",
    "    for i in range(0, len(documents), max_batch_size):\n",
    "        batch_docs = documents[i:i + max_batch_size]\n",
    "        batch_embeddings = embeddings[i:i + max_batch_size]\n",
    "        batch_ids = ids[i:i + max_batch_size]\n",
    "\n",
    "        # Perform the upsert operation for each batch\n",
    "        collection.upsert(\n",
    "            ids=batch_ids,\n",
    "            embeddings=batch_embeddings.tolist(),\n",
    "            documents=batch_docs\n",
    "        )\n",
    "\n",
    "    return collection, embedder\n",
    "\n",
    "\n",
    "def __pipline_gen(model_id = \"meta-llama/Llama-3.2-3B-Instruct\", \n",
    "                bits_rate=4, device_map={\"\": \"cuda:0\"}, torch_dtype=torch.float32,\n",
    "                pipeline_task=\"text-generation\"):\n",
    "    quant_config = BitsAndBytesConfig(bit_rate=bits_rate)\n",
    "    llm_model_llama = AutoModelForCausalLM.from_pretrained(\n",
    "        model_id,\n",
    "        quantization_config=quant_config,\n",
    "        device_map=device_map,  # Adjust as needed\n",
    "        torch_dtype=torch_dtype,  # Use mixed precision\n",
    "    )\n",
    "\n",
    "    tokenizer_llama = AutoTokenizer.from_pretrained(model_id)\n",
    "\n",
    "    pipeline_llama = transformers.pipeline(\n",
    "        task=pipeline_task,\n",
    "        model=llm_model_llama,\n",
    "        tokenizer=tokenizer_llama,\n",
    "        output_hidden_states=True,  # Ensure the model outputs hidden states\n",
    "        # device=0                  # Ensure this matches your GPU device ID\n",
    "    )\n",
    "\n",
    "    tokenizer_llama.pad_token_id = tokenizer_llama.eos_token_id\n",
    "    llm_model_llama.pad_token_id = tokenizer_llama.eos_token_id\n",
    "\n",
    "    llm_model_llama = pipeline_llama.model; tokenizer_llama = pipeline_llama.tokenizer \n",
    "    llm_model_llama.eval()\n",
    "    return pipeline_llama\n",
    "\n",
    "def __retrieve_relevant_documents(query, collection, embedder, k=5):\n",
    "    query_embedding = embedder.encode([query])\n",
    "    results = collection.query(\n",
    "        query_embeddings=query_embedding.tolist(),\n",
    "        n_results=k\n",
    "    )\n",
    "    documents = [doc[0] if isinstance(doc, list) else doc for doc in results['documents']]\n",
    "    \n",
    "    return documents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Together AI call for RAG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from together import Together\n",
    "client = Together(api_key=KEY_TOGETHERAI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def qa_together(query = \"ohh hii\", model_name = \"meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo\",\n",
    "                collection = None, embedder = None,\n",
    "                max_token = 512,\n",
    "                max_new_tokens = 1024,\n",
    "                temperature = 0.7,\n",
    "                top_p = 0.7,\n",
    "                top_k = 50,\n",
    "                stop = [\"<|eot_id|>\",\"<|eom_id|>\"],\n",
    "                repitition_penalty = 1,\n",
    "                stream = True,\n",
    "                answer = None,\n",
    "                file_name = \"output.txt\",\n",
    "                verbose = 0):        \n",
    "    try:\n",
    "        relevant_docs = __retrieve_relevant_documents(query, collection=collection, embedder=embedder, k=5)\n",
    "        context = \"\\n\".join(relevant_docs)\n",
    "        prompt = f\"Context: {context}\\n\\nQ: {query}\\nA:\"\n",
    "        response = client.chat.completions.create(\n",
    "                model=model_name,\n",
    "                messages=[\n",
    "                    {\n",
    "                    \"role\": \"user\",\n",
    "                    \"content\": f\"{prompt}\"\n",
    "                    }\n",
    "                ],\n",
    "                max_new_tokens=max_new_tokens,\n",
    "        )\n",
    "        if verbose > 1: \n",
    "            with open(file_name, \"a\") as f: \n",
    "                f.write(f\"PROMT{{{prompt}}}\\n\")\n",
    "                f.write(f\"CORRECT_ANSWER{{{answer}}}\\n\")\n",
    "                f.write(response.choices[0].message.content)\n",
    "        if verbose > 0: print(response.choices[0].message.content)\n",
    "        return response.choices[0].message.content\n",
    "    except Exception as e: \n",
    "        print(f\"Error in response {e}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RAG Collection creation and populating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "collection, embedder = __add_documents_to_collection(\n",
    "                            documents =  __process_documents(__load_documents(DUMP_FOLD_PATH)), \n",
    "                            rag_objects=__create_collection(),\n",
    "                            # max_batch_size=10**12\n",
    "                        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # pickle dump collection, embedder    \n",
    "# import pickle\n",
    "# with open('collection.pkl', 'wb') as f:\n",
    "#     pickle.dump(collection, f)\n",
    "# with open('embedder.pkl', 'wb') as f:\n",
    "#     pickle.dump(embedder, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # load collection, embedder\n",
    "# import pickle\n",
    "# with open('collection.pkl', 'rb') as f:\n",
    "#     collection = pickle.load(f)\n",
    "# with open('embedder.pkl', 'rb') as f:\n",
    "#     embedder = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I'm assuming the context is related to a B.Tech. (CSD) degree, which is likely related to Computer Science and Design. However, without more information about the specific institution or course, I'll provide a general outline of common requirements for a B.Tech. degree:\n",
      "\n",
      "For a B.Tech. (CSD) degree, a student must satisfy the following requirements:\n",
      "\n",
      "1. **Credit Requirements**: Complete a minimum of 160-180 credits, depending on the institution, with a distribution of credits across various subjects, including core, elective, and lab courses.\n",
      "2. **Core Courses**: Complete all the core courses specified by the institution, which typically include subjects like mathematics, computer science, and engineering fundamentals.\n",
      "3. **Elective Courses**: Complete a set of elective courses, which may include specializations like data science, artificial intelligence, machine learning, or software engineering.\n",
      "4. **Lab Courses**: Complete a set of lab courses, which provide hands-on experience in programming, software development, and other technical skills.\n",
      "5. **Project Work**: Complete a project or thesis, which demonstrates the student's ability to apply theoretical knowledge to real-world problems.\n",
      "6. **Internship**: Complete an internship or industrial training, which provides practical experience in the industry.\n",
      "7. **Semester-wise Performance**: Maintain a minimum cumulative grade point average (CGPA) or a minimum percentage of marks in each semester.\n",
      "8. **Attendance**: Maintain a minimum attendance requirement, which may vary depending on the institution.\n",
      "9. **Degree Requirements**: Complete all the degree requirements specified by the institution, which may include a set of core courses, electives, and a project or thesis.\n",
      "10. **University Regulations**: Satisfy all the university regulations, including those related to academic integrity, plagiarism, and other policies.\n",
      "\n",
      "Please note that these requirements may vary depending on the institution and the specific course. It's essential to check with the institution's website or consult with the academic advisor for the most up-to-date and accurate information.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"I'm assuming the context is related to a B.Tech. (CSD) degree, which is likely related to Computer Science and Design. However, without more information about the specific institution or course, I'll provide a general outline of common requirements for a B.Tech. degree:\\n\\nFor a B.Tech. (CSD) degree, a student must satisfy the following requirements:\\n\\n1. **Credit Requirements**: Complete a minimum of 160-180 credits, depending on the institution, with a distribution of credits across various subjects, including core, elective, and lab courses.\\n2. **Core Courses**: Complete all the core courses specified by the institution, which typically include subjects like mathematics, computer science, and engineering fundamentals.\\n3. **Elective Courses**: Complete a set of elective courses, which may include specializations like data science, artificial intelligence, machine learning, or software engineering.\\n4. **Lab Courses**: Complete a set of lab courses, which provide hands-on experience in programming, software development, and other technical skills.\\n5. **Project Work**: Complete a project or thesis, which demonstrates the student's ability to apply theoretical knowledge to real-world problems.\\n6. **Internship**: Complete an internship or industrial training, which provides practical experience in the industry.\\n7. **Semester-wise Performance**: Maintain a minimum cumulative grade point average (CGPA) or a minimum percentage of marks in each semester.\\n8. **Attendance**: Maintain a minimum attendance requirement, which may vary depending on the institution.\\n9. **Degree Requirements**: Complete all the degree requirements specified by the institution, which may include a set of core courses, electives, and a project or thesis.\\n10. **University Regulations**: Satisfy all the university regulations, including those related to academic integrity, plagiarism, and other policies.\\n\\nPlease note that these requirements may vary depending on the institution and the specific course. It's essential to check with the institution's website or consult with the academic advisor for the most up-to-date and accurate information.\""
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"IIITD B.Tech. program is broadly divided in two halves, tell about them ?\"\n",
    "query = \"What are reqirement for CSD program for graduation in IIITD ?\"\n",
    "query = \"What are requirements for Graduation; For a B.Tech. (CSD) degree, a student must satisfy which all the following requirements:\"\n",
    "qa_together(query = query, collection=collection, embedder=embedder, max_new_tokens=32, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pipeline = __pipline_gen()\n",
    "\n",
    "# def generate_rag_answer(query, pipe=pipeline, \n",
    "#                         max_length=80, max_new_tokens=75, temperature=0.01, top_p=1.0, top_k=50,\n",
    "#                         verbose=0, \n",
    "#                         ):\n",
    "#     relevant_docs = __retrieve_relevant_documents(query, collection=collection, embedder=embedder, k=5)\n",
    "#     context = \"\\n\".join(relevant_docs) \n",
    "#     prompt = f\"Context: {context}\\n\\nQ: {query}\\nA:\"\n",
    "#     result = pipe(\n",
    "#         prompt,\n",
    "#         # max_length=max_length,\n",
    "#         max_new_tokens=max_new_tokens,\n",
    "#         do_sample=True,\n",
    "#         temperature=temperature,\n",
    "#         top_p=top_p,\n",
    "#         top_k=top_k\n",
    "#     )\n",
    "#     answer = result[0]['generated_text'].replace(prompt, '').strip()\n",
    "#     if verbose: print(f\"Raw answer: {result}\")\n",
    "#     return answer\n",
    "\n",
    "# generate_rag_answer(\"Tell me about academic calender?\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tmp_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}