CancerCareAI.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "upC8NRqaGjvA"
      ],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5259526d84a949c49876d3023f1a93fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5c7925792d36448f9b203f2a1f4e0029",
              "IPY_MODEL_0ac0ef82c7784e4f9dbe34d631ed4572",
              "IPY_MODEL_ecf0bd2b06c54eb6b7f3822e9320922a"
            ],
            "layout": "IPY_MODEL_6b6e65925dd54c0f919605472ca63de6"
          }
        },
        "5c7925792d36448f9b203f2a1f4e0029": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dbcecf58452d4602accb6e5e6ff091c6",
            "placeholder": "​",
            "style": "IPY_MODEL_5435ed62749445748dbe751d2905f8db",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "0ac0ef82c7784e4f9dbe34d631ed4572": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dfea50bdbb254c37b0dfee8e3502d8b3",
            "max": 4,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_081cc0769ade4be9a862110d6d50ea91",
            "value": 4
          }
        },
        "ecf0bd2b06c54eb6b7f3822e9320922a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f10b461eef9a42cb8347d17a4348044d",
            "placeholder": "​",
            "style": "IPY_MODEL_5bd0ffa967924ac09d813fff391fb4eb",
            "value": " 4/4 [00:42&lt;00:00, 21.08s/it]"
          }
        },
        "6b6e65925dd54c0f919605472ca63de6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dbcecf58452d4602accb6e5e6ff091c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5435ed62749445748dbe751d2905f8db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dfea50bdbb254c37b0dfee8e3502d8b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "081cc0769ade4be9a862110d6d50ea91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f10b461eef9a42cb8347d17a4348044d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5bd0ffa967924ac09d813fff391fb4eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/403errors/CancerCareAI/blob/main/CancerCareAI.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## AI-powered extraction of cancer-related patient data.\n",
        "\n",
        "NOTE: MUST RUN IN **T4 GPU** BECAUSE THE BITSANDBYTES MODULE, USED FOR DOWNLOADING QUANTISED MODEL IS ONLY SUPPORTED REQUIRES GPU ⚠️\n",
        "\n",
        "NOTE: IT IS POSSIBLE THAT THE MODEL DOESN'T LOAD IN ONE RUN, IF IT GIVES \"BITSANDBYTES\" RELATED ISSUES, FOLLOW BELOW STEPS:\n",
        "- RUNTIME > RESTART SESSION\n",
        "- RUNTIME > RUN ALL\n",
        "\n",
        "DONE ✅\n",
        "\n",
        "\n",
        "\n",
        "**Please checkout [GitHub Repo](https://github.com/403errors/CancerCareAI) for more details and demo videos of project.**"
      ],
      "metadata": {
        "id": "upC8NRqaGjvA"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Project Setup and Data Loading\n",
        "This block handles the initial setup and data loading for the project. It performs the following key functions:\n",
        "\n",
        "- Dependency Installation: Installs necessary Python libraries using pip.\n",
        "\n",
        "  This includes:\n",
        "  - sentence-transformers: For semantic similarity calculations.\n",
        "rank_bm25: For implementing the BM25 ranking algorithm (keyword-based retrieval).\n",
        "  - pandas: For data manipulation and creating DataFrames.\n",
        "  - nltk: Natural Language Toolkit, used here for sentence tokenization.\n",
        "  - bitsandbytes: For loading quantized large language models.\n",
        "  - accelerate: A Hugging Face library for distributed training and inference.\n",
        "  - optimum: Hugging Face library for optimizing models.\n",
        "\n",
        "- Library Imports: Imports the necessary modules from the installed libraries. This includes modules for semantic search, BM25 ranking, data handling, and working with large language models.\n",
        "\n",
        "- Data Loading (load_data_from_github function):\n",
        "Fetches JSON data files from a specified GitHub repository.\n",
        "  - Takes the repository URL and a list of filenames as input.\n",
        "  - Uses the requests library to make HTTP GET requests to retrieve each file.\n",
        "  - Uses response.raise_for_status() to handle potential HTTP errors (e.g., 404 Not Found).\n",
        "  - Parses the JSON response using response.json().\n",
        "  - Returns a dictionary where keys are filenames and values are the loaded JSON data.\n",
        "  - NLTK Downloads: Downloads NLTK resources\n",
        "    - 'punkt': for setence tokenization\n",
        "    - 'punkt_tab': fallback tokeinzer"
      ],
      "metadata": {
        "id": "sH2JYxW4uFyR"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### pip installation"
      ],
      "metadata": {
        "id": "Swg6bqhSuFyS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Install necessary libraries (if not already installed in your Colab environment)\n",
        "!pip install -q sentence-transformers rank_bm25 pandas nltk\n",
        "!pip install -q --no-cache-dir bitsandbytes\n",
        "!pip install -q accelerate optimum"
      ],
      "metadata": {
        "id": "Jp51nKtHuFyS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Imports"
      ],
      "metadata": {
        "id": "YK4ZxCAuuFyS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "import json\n",
        "import requests\n",
        "from sentence_transformers import SentenceTransformer, CrossEncoder, util\n",
        "import torch\n",
        "from rank_bm25 import BM25Okapi\n",
        "import pandas as pd\n",
        "from tqdm.auto import tqdm\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
        "import nltk\n",
        "nltk.download('punkt')  # Download the Punkt sentence tokenizer\n",
        "nltk.download('punkt_tab')  # It works instead of punkt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0786cb5c-35e6-4423-84e7-b4e004dfd4d6",
        "id": "PBoqkGYmuFyS"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package punkt_tab to /root/nltk_data...\n",
            "[nltk_data]   Package punkt_tab is already up-to-date!\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Loading"
      ],
      "metadata": {
        "id": "pLy6pT-AuFyT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def load_data_from_github(repo_url, filenames):\n",
        "    \"\"\"Loads JSON data files from a GitHub repository.\n",
        "\n",
        "    Args:\n",
        "        repo_url: Base URL of the GitHub repository's data directory.\n",
        "        filenames: List of filenames to load.\n",
        "\n",
        "    Returns:\n",
        "        A dictionary where keys are filenames and values are the loaded JSON data.\n",
        "    \"\"\"\n",
        "    data = {}\n",
        "    for filename in filenames:\n",
        "        file_url = f\"{repo_url}/{filename}\"\n",
        "        response = requests.get(file_url)\n",
        "        response.raise_for_status()  # Raise an exception for bad status codes\n",
        "        data[filename] = response.json()\n",
        "    return data"
      ],
      "metadata": {
        "id": "5m8cT0M6uFyT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Task 1 - Information Retrieval (Pipeline)\n",
        "This block implements the information retrieval pipeline, combining keyword-based and semantic search techniques. It includes the following components:\n",
        "\n",
        "- create_passages(patient_data):\n",
        "Transforms the raw JSON data (loaded in the previous block) into a list of dictionaries.\n",
        "  - Each dictionary represents a single sentence extracted from the docText field of the original data. Crucially, it uses nltk.sent_tokenize to split the text into sentences, rather than treating entire documents as passages.\n",
        "  - Includes metadata: docTitle, docDate, and patient_file are preserved alongside the docText (now a single sentence). This metadata is important for presenting results and filtering.\n",
        "\n",
        "- bm25_ranking(query, passages, tokenizer_bm25):\n",
        "Implements the BM25 (Best Matching 25) ranking algorithm. BM25 is a classic information retrieval algorithm that ranks documents based on the frequency of query terms within each document, adjusted for document length and term frequency in the entire corpus.\n",
        "  - Takes a query string, a list of passages (dictionaries with a \"docText\" key), and a tokenizer_bm25 function as input.\n",
        "  - Uses the provided tokenizer_bm25 to tokenize both the query and the passages. A simple tokenizer that splits on spaces is used.\n",
        "  - Creates a BM25Okapi object from the rank_bm25 library.\n",
        "  - Calculates BM25 scores for the query against each passage.\n",
        "  - Returns a list of (passage, score) tuples, sorted by score in descending order.\n",
        "\n",
        "- semantic_search(query, passages, model_name=\"all-MiniLM-L6-v2\"):\n",
        "Performs semantic search using Sentence Transformers, a library for generating dense vector representations (embeddings) of text.\n",
        "  - Takes a query string, a list of passages, and an optional model_name (defaulting to \"all-MiniLM-L6-v2\") as input.\n",
        "  - Loads the specified Sentence Transformer model. \"all-MiniLM-L6-v2\" is a pre-trained model known for its good balance of speed and accuracy.\n",
        "  - Generates embeddings for the query and all passages using model.encode().\n",
        "  - Uses util.semantic_search() to efficiently find the passages with embeddings most similar to the query embedding (using cosine similarity).\n",
        "  - Returns a list of (passage, score) tuples, sorted by similarity score in descending order.\n",
        "\n",
        "- rerank_with_crossencoder(query, passages, model_name=\"cross-encoder/ms-marco-MiniLM-L-6-v2\"):\n",
        "  - Reranks a subset of passages using a CrossEncoder model, which is generally more accurate than the Bi-Encoder used in semantic_search but computationally more expensive.\n",
        "  - Takes a query, a list of passages, and an optional model_name (defaulting to \"cross-encoder/ms-marco-MiniLM-L-6-v2\") as input.\n",
        "  - Loads the specified CrossEncoder model. \"cross-encoder/ms-marco-MiniLM-L-6-v2\" is a model trained on the MS MARCO passage ranking dataset.\n",
        "  - CrossEncoders take a (query, passage) pair as input and directly predict a relevance score. This is different from Bi-Encoders, which generate separate embeddings for the query and passage and then calculate similarity.\n",
        "  - Calculates scores using model.predict().\n",
        "  - Returns a list of (passage, score) tuples, sorted by score in descending order.\n",
        "\n",
        "- filter_sentence(sentence):\n",
        "Filters out the irrelevant sentences that contains adminstrative details.\n",
        "Takes setence as an argument.\n",
        "  - Discard sentences containing exclude_patterns using regex.\n",
        "  - Return True if pattern not found.\n",
        "\n",
        "- combined_retrieval(query, passages, bm25_weight=0.4, semantic_weight=0.3, crossencoder_weight=0.3):\n",
        "This is the core function that orchestrates the entire retrieval pipeline.\n",
        "Takes a query, a list of passages, and optional weights for each ranking method as input.\n",
        "  - Calls bm25_ranking and semantic_search to get initial rankings.\n",
        "  - Filters the results of both BM25 and semantic search, keeping only the top top_n (default 20) passages from each. This is a crucial optimization step. It avoids running the computationally expensive CrossEncoder on all passages.\n",
        "  - Also applies filter_sentence() to remove administrative/irrelevant sentences.\n",
        "  - Combines the top passages from BM25 and semantic search, removing duplicates by using a set to track unique docText values. This ensures that the same sentence isn't included multiple times.\n",
        "  - Calls rerank_with_crossencoder to rerank the combined, filtered list of passages.\n",
        "  - Normalizes the scores from each method (BM25, semantic search and CrossEncoder) to a range of 0 to 1. This is important because the raw scores from different methods are not directly comparable. Min-max normalization is used.\n",
        "  - Combines the normalized scores using a weighted average: combined_score = bm25_score * bm25_weight + semantic_score * semantic_weight + crossencoder_score * crossencoder_weight. The weights allow for tuning the relative importance of each ranking method.\n",
        "  - Returns a list of (passage, combined_score) tuples, sorted by the combined score in descending order. The passages retain all original metadata."
      ],
      "metadata": {
        "id": "lqCYCp4XuFyT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def create_passages(patient_data):\n",
        "    \"\"\"Creates a list of SENTENCES (passages) from the patient data.\"\"\"\n",
        "    passages = []\n",
        "    for filename, patient_records in patient_data.items():\n",
        "        for record in patient_records:\n",
        "            # Split the docText into sentences using nltk.sent_tokenize\n",
        "            sentences = nltk.sent_tokenize(record[\"docText\"])\n",
        "            for sentence in sentences:\n",
        "                passages.append({\n",
        "                    \"docText\": sentence,  # Now, docText is a single sentence\n",
        "                    \"docTitle\": record[\"docTitle\"],\n",
        "                    \"docDate\": record[\"docDate\"],\n",
        "                    \"patient_file\": filename\n",
        "                })\n",
        "    return passages\n",
        "\n",
        "\n",
        "def bm25_ranking(query, passages, tokenizer_bm25):\n",
        "  \"\"\"\n",
        "    Ranks passages using BM25.\n",
        "\n",
        "    Args:\n",
        "      query: search query (String)\n",
        "      passages: a list of dictionaries; dictionaries must contain the \"docText\" key\n",
        "      tokenizer_bm25: A tokenizer suitable for BM25 (e.g., splitting on spaces).\n",
        "\n",
        "    Returns:\n",
        "      List of (passage, score) tuples, sorted by score (highest first).\n",
        "  \"\"\"\n",
        "  tokenized_corpus = [tokenizer_bm25(p[\"docText\"]) for p in passages]\n",
        "  bm25_model = BM25Okapi(tokenized_corpus)\n",
        "  tokenized_query = tokenizer_bm25(query)\n",
        "  doc_scores = bm25_model.get_scores(tokenized_query)\n",
        "  # Combine passages and scores\n",
        "  passage_scores = list(zip(passages, doc_scores))\n",
        "  # Sort by score (descending)\n",
        "  passage_scores.sort(key=lambda x: x[1], reverse=True)\n",
        "  return passage_scores\n",
        "\n",
        "\n",
        "def semantic_search(query, passages, model_name=\"all-MiniLM-L6-v2\"):\n",
        "    \"\"\"Performs semantic search using Sentence Transformers.\n",
        "\n",
        "    Args:\n",
        "        query: The search query.\n",
        "        passages: A list of dictionaries, where each dictionary must contain at least \"docText\".\n",
        "        model_name: The Sentence Transformer model to use.\n",
        "\n",
        "    Returns:\n",
        "        A list of (passage, score) tuples, sorted by similarity score (highest first).\n",
        "    \"\"\"\n",
        "    model = SentenceTransformer(model_name)\n",
        "    corpus_embeddings = model.encode([p[\"docText\"] for p in passages], convert_to_tensor=True)\n",
        "    query_embedding = model.encode(query, convert_to_tensor=True)\n",
        "    hits = util.semantic_search(query_embedding, corpus_embeddings, top_k=10)[0] #top_k can be adjusted\n",
        "    results = []\n",
        "    for hit in hits:\n",
        "        results.append((passages[hit['corpus_id']], hit['score']))\n",
        "    return results\n",
        "\n",
        "def rerank_with_crossencoder(query, passages, model_name=\"cross-encoder/ms-marco-MiniLM-L-6-v2\"):\n",
        "    \"\"\"Reranks passages using a CrossEncoder model.\n",
        "\n",
        "    Args:\n",
        "      query: the search query.\n",
        "      passages: A list of dictionaries; must contain the \"docText\" key.\n",
        "      model_name: the CrossEncoder model to use.\n",
        "\n",
        "    Returns:\n",
        "      List of (passage, score) tuples, sorted by score (highest first).\n",
        "    \"\"\"\n",
        "    model = CrossEncoder(model_name)\n",
        "    scores = model.predict([(query, p[\"docText\"]) for p in passages])\n",
        "    passage_scores = list(zip(passages, scores))\n",
        "    passage_scores.sort(key=lambda x: x[1], reverse=True)\n",
        "    return passage_scores\n",
        "\n",
        "def filter_sentence(sentence):\n",
        "    \"\"\"Filters out sentences that are likely administrative or irrelevant.\"\"\"\n",
        "    # List of patterns to exclude (case-insensitive)\n",
        "    exclude_patterns = [\n",
        "        r\"Patient Name:\",\n",
        "        r\"Date of Birth:\",\n",
        "        r\"Location:\",\n",
        "        r\"Purpose:\",\n",
        "        r\"Follow-up:\",\n",
        "        r\"Conclusion:\",\n",
        "        r\"Next Steps:\",\n",
        "        r\"Date of Visit:\",\n",
        "        r\":\", # Remove all the sentences that contain \":\"\n",
        "    ]\n",
        "    for pattern in exclude_patterns:\n",
        "        if re.search(pattern, sentence, re.IGNORECASE):\n",
        "            return False  # Discard the sentence\n",
        "    return True  # Keep the sentence\n",
        "\n",
        "def combined_retrieval(query, passages, bm25_weight=0.4, semantic_weight=0.3, crossencoder_weight=0.3):\n",
        "    \"\"\"Combines BM25, semantic search, and cross-encoder reranking.\n",
        "    Args:\n",
        "        query:\n",
        "        passages:\n",
        "        bm25_weight:\n",
        "        semantic_weight:\n",
        "        crossencoder_weight:\n",
        "    Returns:\n",
        "       List of (passage, combined_score) tuples\n",
        "    \"\"\"\n",
        "    # Simple tokenizer for BM25 (split on spaces)\n",
        "    tokenizer_bm25 = lambda text: text.lower().split()\n",
        "\n",
        "    # 1. BM25 Ranking\n",
        "    bm25_results = bm25_ranking(query, passages, tokenizer_bm25)\n",
        "\n",
        "    # 2. Semantic Search\n",
        "    semantic_results = semantic_search(query, passages)\n",
        "\n",
        "    # 3.  Filter to top N from BM25 and Semantic Search before Cross-Encoding\n",
        "    top_n = 20  # Adjust as needed\n",
        "\n",
        "    # --- Filtering before Cross-Encoding ---\n",
        "    bm25_top_n = [passage for passage, _ in bm25_results[:top_n] if filter_sentence(passage[\"docText\"])]\n",
        "    semantic_top_n = [passage for passage, _ in semantic_results[:top_n] if filter_sentence(passage[\"docText\"])]\n",
        "\n",
        "    # Use a set to track unique docText values\n",
        "    unique_doc_texts = set()\n",
        "    combined_top_passages = []\n",
        "\n",
        "    for passage in bm25_top_n + semantic_top_n:\n",
        "        doc_text = passage[\"docText\"]  # Extract the unique text identifier\n",
        "        if doc_text not in unique_doc_texts:\n",
        "            unique_doc_texts.add(doc_text)\n",
        "            combined_top_passages.append(passage)  # Append the full passage dict\n",
        "\n",
        "    # 4. Cross-Encoder Reranking (on the combined top passages)\n",
        "    crossencoder_results = rerank_with_crossencoder(query, combined_top_passages)\n",
        "\n",
        "\n",
        "    # 5. Normalize and Combine Scores (using a dictionary for easier lookup)\n",
        "    def normalize_scores(results):\n",
        "        if not results:\n",
        "            return {}\n",
        "        scores = [score for _, score in results]\n",
        "        min_score = min(scores)\n",
        "        max_score = max(scores)\n",
        "        if max_score == min_score:  # Avoid division by zero\n",
        "            return {passage[\"docText\"]: 0.5 for passage, _ in results}  #Give them all a neutral score\n",
        "        return {passage[\"docText\"]: (score - min_score) / (max_score - min_score) for passage, score in results}\n",
        "\n",
        "    bm25_scores = normalize_scores(bm25_results)\n",
        "    semantic_scores = normalize_scores(semantic_results)\n",
        "    crossencoder_scores = normalize_scores(crossencoder_results)\n",
        "\n",
        "    # Combine (using docText as the key, since it's unique within the same query)\n",
        "    combined_scores = {}\n",
        "    for passage, _ in crossencoder_results:  # Iterate through crossencoder results as the base\n",
        "        doc_text = passage[\"docText\"]\n",
        "        combined_score = (\n",
        "            bm25_scores.get(doc_text, 0) * bm25_weight +  # Use .get() to handle missing keys\n",
        "            semantic_scores.get(doc_text, 0) * semantic_weight +\n",
        "            crossencoder_scores.get(doc_text, 1) * crossencoder_weight # crossencoder_weight default to 1 as it contains all.\n",
        "        )\n",
        "        combined_scores[doc_text] = combined_score\n",
        "\n",
        "    # Convert back to a list of (passage, score) tuples, preserving passage data\n",
        "    final_results = []\n",
        "    for passage, _ in crossencoder_results: # We want the order from the cross-encoder\n",
        "      if passage[\"docText\"] in combined_scores: # This check should always pass\n",
        "        final_results.append((passage, combined_scores[passage[\"docText\"]]))\n",
        "\n",
        "    return final_results"
      ],
      "metadata": {
        "id": "c4y8C_utuFyT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Task 2 - Medical Data Extraction (LLM-based Pipeline)\n",
        "\n",
        "This is where we use the Qwen 2.5-7B-Chat model to extract structured data. We'll create a function to generate the prompt and another to process the model's output\n",
        "\n",
        "**Explanation:**\n",
        "\n",
        "* setup_qwen_model: This function loads the Qwen model and tokenizer, applying the 4-bit quantization to reduce memory usage. This is the same code provided in the README, but encapsulated in a function for reusability. We also make sure to move the model and inputs to the correct device (GPU if available, otherwise CPU). Also, set pad_token to eos_token.\n",
        "\n",
        "* generate_prompt: This function creates the prompt that will be fed to the LLM. It includes:\n",
        "    * Clear Instructions: It tells the model its role (\"medical information extraction expert\") and what to extract.\n",
        "    * Passage Context: It includes the passage_text.\n",
        "    * Structured Output Format: It explicitly defines the JSON structure we want, including examples of each field. This is crucial for reliable JSON output.\n",
        "    * Handling Null Values: The instructions clearly explain that if a particular data point can't be found, null should be used.\n",
        "\n",
        "* extract_information: This function does the following:\n",
        "    * Tokenization: It tokenizes the prompt using the Qwen tokenizer.\n",
        "    * Inference: It calls model.generate to generate the output. We use:\n",
        "        * max_new_tokens: Limits the length of the generated text.\n",
        "        * do_sample=False: Uses greedy decoding (taking the most likely token at each step). This makes the output deterministic (same input always gives the same output).\n",
        "        * temperature=0.1: We use a low temperature to make the model less \"creative\" and more likely to stick to the instructions.\n",
        "        * top_k=5: Limits the model to consider only the top 5 most likely tokens at each step. This further reduces randomness.\n",
        "        * with torch.no_grad(): Disables gradient calculation, saving memory and speeding up inference.\n",
        "        * pad_token_id=tokenizer.eos_token_id:Set pad_token_id.\n",
        "\n",
        "    * Decoding: It decodes the generated output using the tokenizer.\n",
        "\n",
        "    * JSON Extraction: It extracts the JSON part from the output. This is the most critical part. We use find('{') and rfind('}') + 1 to locate the JSON object within the LLM's response, handling cases where the model might add extra text before or after the JSON.\n",
        "    * Error handling: Robust error handling is included using try-except blocks to catch potential json.JSONDecodeError (if the output isn't valid JSON) or ValueError. This makes the code much more resilient. The raw LLM output is printed for debugging purposes.\n",
        "* Deterministic Output: By setting do_sample=False, temperature, and top_k, we encourage the model to produce consistent, deterministic output, which is essential for reliable data extraction.\n",
        "\n"
      ],
      "metadata": {
        "id": "YeJit6tQuFyU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def setup_qwen_model():\n",
        "    \"\"\"Sets up the Qwen model, checking for CUDA and using 4-bit quantization if available.\"\"\"\n",
        "    model_name = \"Qwen/Qwen2.5-7B-Instruct-1M\"\n",
        "    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "    print(f\"Using device: {device}\")\n",
        "\n",
        "    if device == \"cuda\":\n",
        "        quantization_config = BitsAndBytesConfig(\n",
        "            load_in_4bit=True,\n",
        "            bnb_4bit_compute_dtype=torch.float16,\n",
        "            bnb_4bit_use_double_quant=True\n",
        "        )\n",
        "        model = AutoModelForCausalLM.from_pretrained(\n",
        "            model_name,\n",
        "            quantization_config=quantization_config,\n",
        "            device_map=\"auto\",\n",
        "            use_safetensors=True,\n",
        "            low_cpu_mem_usage=True,\n",
        "            trust_remote_code=True\n",
        "        )\n",
        "    else:\n",
        "        print(\"CPU usage requires significant RAM; quantization (GPU only) is recommended.\")\n",
        "        model = None  # Model will not be available\n",
        "\n",
        "    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)\n",
        "    tokenizer.pad_token = tokenizer.eos_token\n",
        "    return model, tokenizer, device"
      ],
      "metadata": {
        "id": "XCqM90B1uFyU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_prompt(passage_text):\n",
        "    \"\"\"Generates a structured prompt for extracting specific medical information in JSON format.\"\"\"\n",
        "    prompt = f\"\"\"<|im_start|>system\n",
        "    You are a medical information extraction system. Extract structured data from patient EHR notes and return it as a **single JSON object** containing exactly two arrays: `diagnosis_characteristics` and `cancer_related_medications`. Do not include any other keys at the top level.\n",
        "\n",
        "    ### **Guidelines:**\n",
        "    - Output **ONLY** strictly valid JSON—no extra text, explanations, or remarks.\n",
        "    - If information is absent, use `null` (not empty strings).\n",
        "    - Maintain the exact JSON structure and formatting.\n",
        "\n",
        "    #### **1. diagnosis_characteristics (array of dictionaries)**\n",
        "    Each dictionary represents a primary cancer diagnosis and must include:\n",
        "    - `primary_cancer_condition` (string): Type of cancer (e.g., \"Breast Cancer\").\n",
        "    - `diagnosis_date` (string, \"MM-DD-YYYY\"): Earliest diagnosis date.\n",
        "    - `histology` (array of strings): Tumor histological subtype(s).\n",
        "    - `stage` (dictionary): TNM and group stage details:\n",
        "        - `T` (string): Tumor size/extent.\n",
        "        - `N` (string): Lymph node involvement.\n",
        "        - `M` (string): Metastasis status.\n",
        "        - `group_stage` (string): Overall stage (e.g., \"Stage IIB\").\n",
        "\n",
        "    #### **2. cancer_related_medications (array of dictionaries)**\n",
        "    Each dictionary represents a prescribed cancer-related medication:\n",
        "    - `medication_name` (string): Name of the medication.\n",
        "    - `start_date` (string, \"MM-DD-YYYY\"): Start date of the medication.\n",
        "    - `end_date` (string, \"MM-DD-YYYY\" or `null` if ongoing): End date.\n",
        "    - `intent` (string): Purpose of prescription.\n",
        "    <|im_end|>\n",
        "\n",
        "    <|im_start|>user\n",
        "\n",
        "    ### **Example Input:**\n",
        "    *\"Patient was diagnosed with Stage IIB breast cancer, ER+, PR+, HER2-. Diagnosis date: 03-10-2024. Pathology showed invasive ductal carcinoma. Patient started on Letrozole 2.5mg daily starting 03-15-2024 as adjuvant therapy.\"*\n",
        "\n",
        "    ### **Expected Output:**\n",
        "    ```json\n",
        "    {{\n",
        "        \"diagnosis_characteristics\": [\n",
        "            {{\n",
        "                \"primary_cancer_condition\": \"breast cancer\",\n",
        "                \"diagnosis_date\": \"03-10-2024\",\n",
        "                \"histology\": [\"invasive ductal carcinoma\"],\n",
        "                \"stage\": {{\n",
        "                    \"T\": null,\n",
        "                    \"N\": null,\n",
        "                    \"M\": null,\n",
        "                    \"group_stage\": \"Stage IIB\"\n",
        "                }}\n",
        "            }}\n",
        "        ],\n",
        "        \"cancer_related_medications\": [\n",
        "            {{\n",
        "                \"medication_name\": \"Letrozole\",\n",
        "                \"start_date\": \"03-15-2024\",\n",
        "                \"end_date\": null,\n",
        "                \"intent\": \"adjuvant therapy\"\n",
        "            }}\n",
        "        ]\n",
        "    }}\n",
        "    ```\n",
        "\n",
        "\n",
        "    Now, extract structured information from the following passage and output ONLY the JSON format:\n",
        "\n",
        "    {passage_text}\n",
        "\n",
        "    <|im_end|><|im_start|>assistant\"\"\"\n",
        "\n",
        "    return prompt.strip()"
      ],
      "metadata": {
        "id": "2cHYoui-uFyU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def extract_information(model, tokenizer, device, passage, max_new_tokens=1024):\n",
        "    \"\"\"Extracts structured JSON information from a medical passage using a Qwen model.\"\"\"\n",
        "\n",
        "    prompt = generate_prompt(passage[\"docText\"])\n",
        "    inputs = tokenizer(prompt, return_tensors=\"pt\").to(device)\n",
        "\n",
        "    try:\n",
        "        with torch.no_grad():\n",
        "            outputs = model.generate(\n",
        "                **inputs,\n",
        "                max_new_tokens=max_new_tokens,\n",
        "                do_sample=False,\n",
        "                temperature=0.2,\n",
        "                top_k=5,\n",
        "                repetition_penalty=1.0,\n",
        "                pad_token_id=tokenizer.eos_token_id\n",
        "            )\n",
        "\n",
        "        decoded_output = tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
        "\n",
        "        # print(f\"decoded_output: {decoded_output}\")\n",
        "\n",
        "        if \"assistant\" in decoded_output:\n",
        "            assistant_response = decoded_output.split(\"assistant\", 1)[1].strip()\n",
        "        else:\n",
        "            assistant_response = decoded_output\n",
        "\n",
        "        # print(f\"assistant_response: {assistant_response}\")\n",
        "\n",
        "        # Step 1: Extract JSON using a stricter regex pattern\n",
        "        matches = re.findall(r'```json\\s*\\n?(.*?)\\n?```', assistant_response, re.DOTALL)\n",
        "        # print(f\"Matches: {matches}\")\n",
        "\n",
        "        if not matches:\n",
        "            print(\"No JSON block found in the text.\")\n",
        "            return None\n",
        "\n",
        "        # # Step 2: Get the last extracted JSON block (assuming the final one is correct)\n",
        "        json_str = matches[-1].strip()\n",
        "\n",
        "        # Step 3: Remove trailing commas (fixes common JSON errors)\n",
        "        json_str = re.sub(r',\\s*([\\]}])', r'\\1', json_str)\n",
        "\n",
        "        # # Step 4: Debug extracted JSON before parsing\n",
        "        # print(\"Extracted JSON String:\\n\", json_str)\n",
        "\n",
        "        # Step 5: Attempt JSON parsing\n",
        "        try:\n",
        "            return json.loads(json_str)\n",
        "        except json.JSONDecodeError as json_err:\n",
        "            print(f\"JSON parsing error: {json_err}\")\n",
        "            print(\"Attempting secondary cleanup...\")\n",
        "\n",
        "        # Step 6: Handle partial JSON recovery if possible\n",
        "        possible_json_lines = json_str.split(\"\\n\")  # Break into lines\n",
        "        for i in range(len(possible_json_lines)):\n",
        "            try:\n",
        "                cleaned_json_str = \"\\n\".join(possible_json_lines[i:])\n",
        "                return json.loads(cleaned_json_str)  # Return first valid JSON found\n",
        "            except json.JSONDecodeError:\n",
        "                continue  # Try next variation\n",
        "\n",
        "        print(\"Final JSON parsing failed.\")\n",
        "        return None\n",
        "\n",
        "    except Exception as e:\n",
        "        print(f\"Error during generation or JSON parsing: {e}\")\n",
        "        return None"
      ],
      "metadata": {
        "id": "a3zm5WSBuFyU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_passage = {\n",
        "    \"docDate\": \"04-03-2025\",\n",
        "    \"docTitle\": \"Pathology Report - Prostate Biopsy Findings\",\n",
        "    \"docText\": \"Patient Name: Paul Henderson\\nDate of Birth: 03/12/1958\\n\\nDate of Report: 04/03/2025\\nPathologist: Dr. William Archer\\n\\nSpecimens:\\nTwelve core biopsy samples from the prostate, labeled according to standard sextant mapping with additional targeted cores in the right peripheral zone.\\n\\nMicroscopic Examination:\\n1. **Right Peripheral Zone Cores**: Malignant cells consistent with adenocarcinoma of the prostate. The glands are crowded and demonstrate prominent nucleoli. Based on the Gleason grading system, the dominant pattern is 3 and the secondary pattern is 4, yielding a Gleason score of 3+4 = 7.\\n2. **Other Cores (Left Peripheral, Base, Apex)**: Several cores show benign prostatic hyperplasia and chronic inflammatory changes. No malignancy identified in these regions.\\n\\nTumor Characteristics:\\n- In the malignant cores, the extent of involvement ranges from 30% to 60% of the tissue examined.\\n- Perineural invasion is noted, commonly seen in prostate cancer but does not necessarily indicate extraprostatic extension.\\n- No definitive evidence of high-grade (Gleason pattern 5) disease in the submitted samples.\\n\\nDiagnosis:\\nProstatic adenocarcinoma, Gleason 7 (3+4), primarily involving the right peripheral zone.\\n\\nComments:\\nA Gleason score of 7 (3+4) indicates an intermediate-grade prostate cancer. Further staging assessments, including imaging and serum markers, may help determine if the disease is organ-confined. Additional data such as PSA density or genomic tests could refine risk stratification. The presence of perineural invasion can correlate with a slightly higher likelihood of extraprostatic extension, but imaging is required to confirm.\\n\\nRecommendation:\\nCorrelate these findings with clinical and radiologic staging. Consider discussing treatment options with the patient, which may include radical prostatectomy, radiation therapy, or potentially active surveillance if certain criteria are met (though many would treat Gleason 7 more definitively).\\n\\nSigned:\\nDr. William Archer, MD\\nDepartment of Pathology, Hillside Labs\\nReport Date: 04/03/2025\\n\\nConclusion:\\nMr. Henderson’s biopsy confirms prostate cancer in the right peripheral zone with an intermediate Gleason score. Multidisciplinary evaluation with urology, radiation oncology, and possibly medical oncology will be necessary to formulate an optimal treatment plan.\\n\"\n",
        "  }"
      ],
      "metadata": {
        "id": "g2EZl-GGuFyU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# model, tokenizer, device = setup_qwen_model()"
      ],
      "metadata": {
        "id": "Ji1W1g6ouFyV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# ans = extract_information(model, tokenizer, device, test_passage, max_new_tokens=1024)\n",
        "# print(ans)"
      ],
      "metadata": {
        "id": "XbaFfgybuFyV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def merge_extractions(existing_data, new_data):\n",
        "    \"\"\"Merges newly extracted data with existing aggregated data.\"\"\"\n",
        "\n",
        "    if new_data is None:  # Handle cases where nothing was extracted\n",
        "        return existing_data\n",
        "\n",
        "    # --- Merge Diagnosis Characteristics ---\n",
        "    if \"diagnosis_characteristics\" in new_data and isinstance(new_data[\"diagnosis_characteristics\"], list):\n",
        "        for new_diagnosis in new_data[\"diagnosis_characteristics\"]:\n",
        "            if new_diagnosis.get(\"primary_cancer_condition\"):\n",
        "                existing_diagnosis_found = False\n",
        "                for existing_diagnosis in existing_data[\"diagnosis_characteristics\"]:\n",
        "                    if existing_diagnosis[\"primary_cancer_condition\"] == new_diagnosis[\"primary_cancer_condition\"]:\n",
        "                        existing_diagnosis_found = True\n",
        "                        # Update fields if new data is more specific or earlier\n",
        "                        if new_diagnosis.get(\"diagnosis_date\") and (not existing_diagnosis.get(\"diagnosis_date\") or new_diagnosis[\"diagnosis_date\"] < existing_diagnosis[\"diagnosis_date\"]):\n",
        "                            existing_diagnosis[\"diagnosis_date\"] = new_diagnosis[\"diagnosis_date\"]\n",
        "                        if new_diagnosis.get(\"histology\") and isinstance(new_diagnosis[\"histology\"], list): # Check for list\n",
        "                            existing_histology = existing_diagnosis.get(\"histology\") or []  # Default to empty list\n",
        "                            existing_diagnosis[\"histology\"] = list(set(existing_histology + new_diagnosis[\"histology\"]))\n",
        "\n",
        "                        if new_diagnosis.get(\"stage\"):\n",
        "                            for key in [\"T\", \"N\", \"M\", \"group_stage\"]:\n",
        "                                if new_diagnosis[\"stage\"].get(key) is not None: # Only update if not null\n",
        "                                    existing_diagnosis[\"stage\"][key] = new_diagnosis[\"stage\"][key]\n",
        "                        break  # Stop after updating the matching condition\n",
        "                if not existing_diagnosis_found:\n",
        "                    existing_data[\"diagnosis_characteristics\"].append(new_diagnosis)\n",
        "\n",
        "\n",
        "    # --- Merge Medications ---\n",
        "    if \"cancer_related_medications\" in new_data and isinstance(new_data[\"cancer_related_medications\"], list):\n",
        "      for new_med in new_data[\"cancer_related_medications\"]:\n",
        "          if new_med.get(\"medication_name\"):  # Ensure medication name exists\n",
        "              existing_med_found = False\n",
        "              for existing_med in existing_data[\"cancer_related_medications\"]:\n",
        "                  if existing_med[\"medication_name\"] == new_med[\"medication_name\"]:\n",
        "                      existing_med_found = True\n",
        "                      # Update existing medication with new info\n",
        "                      if new_med.get(\"start_date\") and (not existing_med[\"start_date\"] or\n",
        "                                                        new_med[\"start_date\"] < existing_med[\"start_date\"]):\n",
        "                          existing_med[\"start_date\"] = new_med[\"start_date\"]\n",
        "                      if new_med.get(\"end_date\") and (not existing_med[\"end_date\"] or\n",
        "                                                      new_med[\"end_date\"] > existing_med[\"end_date\"]):\n",
        "                          existing_med[\"end_date\"] = new_med[\"end_date\"]\n",
        "\n",
        "                      #Combine the existing and new intent\n",
        "                      if new_med.get(\"intent\"):\n",
        "                          existing_med[\"intent\"] = existing_med.get(\"intent\", \"\") + \" \" + new_med.get(\"intent\",\"\")\n",
        "                      break\n",
        "\n",
        "              if not existing_med_found:\n",
        "                  existing_data[\"cancer_related_medications\"].append(new_med)\n",
        "\n",
        "    return existing_data"
      ],
      "metadata": {
        "id": "G-bxzbZ3uFyV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Putting it all Together (Main Execution Block)\n",
        "\n",
        "This block is the main execution point of the script. It orchestrates the entire process, from data loading and user interaction to information retrieval/extraction and output.\n",
        "\n",
        "- if __name__ == \"__main__\": Guard:\n",
        "This standard Python construct ensures that the code within this block only runs when the script is executed directly (e.g., python cancercareai.py), not when it's imported as a module into another script.\n",
        "- Data Loading and Error Handling:\n",
        "  - Loads the patient data from the GitHub repository using the load_data_from_github function (defined in Block 1).\n",
        "  - Includes a critical error check: if patient_data is None:. If data loading fails (e.g., due to network issues), the script prints an error message and exits gracefully, preventing subsequent code from crashing.\n",
        "- Patient Selection (Interactive):\n",
        "  - Presents a list of available patients to the user.\n",
        "  - Extracts patient names from the docText field of the first record in each file using a regular expression (re.search). Handles cases where the name cannot be extracted.\n",
        "  - Prompts the user to enter the name of the patient they want to process.\n",
        "  - Handles cases where the entered name is not found or where multiple files match the name (allowing the user to choose a specific file).\n",
        "  - Creates selected_patient_data, containing only the data for the selected patient. This improves efficiency by focusing on the relevant data.\n",
        "- Mode Selection (Interactive):\n",
        "  Prompts the user to choose between two modes:\n",
        "  - Mode 1 (Query): Performs information retrieval based on a user-provided query (using the pipeline from Block 2).\n",
        "  - Mode 2 (Medical Data Extraction): Extracts structured medical data using the LLM (using the pipeline from Block 3).\n",
        "    - Checks if the model is available on the CPU.\n",
        "- Mode 1: Query (Information Retrieval):\n",
        "  - Prompts the user to enter a query.\n",
        "  - Calls create_passages to convert the selected patient's data into a list of sentence-level passages.\n",
        "  - Calls combined_retrieval (from Block 2) to perform the combined BM25 and semantic search, retrieving the most relevant passages.\n",
        "  - Prints the top 5 retrieved sentences, along with their associated scores.\n",
        "- Mode 2: Medical Data Extraction:\n",
        "  - Initializes an empty dictionary aggregated_data to store the extracted information. This dictionary will accumulate results from multiple documents.\n",
        "  - Iterates through each document (record) in the selected_patient_data. It processes entire documents, not individual sentences, for extraction.\n",
        "  - For each document:\n",
        "    - Creates a passage dictionary containing the docText, docTitle, docDate, and patient_file.\n",
        "    - Calls extract_information (from Block 3) to extract structured data from the document using the LLM.\n",
        "    - Calls merge_extractions to combine the newly extracted data with the aggregated_data. This function handles merging and deduplication of information across multiple documents.\n",
        "    - Prints progress information (which document is being processed).\n",
        "    - After processing all documents, prints the final aggregated_data in JSON format using json.dumps with an indent of 2 for readability.\n",
        "    - Handles the case where no data was extracted, printing an informative message.\n",
        "- Invalid Mode Handling:\n",
        "  - If the user enters an invalid mode choice, prints an error message."
      ],
      "metadata": {
        "id": "sjmzC17TuFyV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "***Mode 2 (Medical Data Extraction) requires at least 6GB VRAM in GPU***\n",
        "\n",
        "❌ CPU only - No GPU acceleration possible\n",
        "\n",
        "✅ T4 GPU - Supports GPU acceleration"
      ],
      "metadata": {
        "id": "1Pg9veTfuFyW"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**NOTE: CUDA 12.x DOESN'T FULLY SUPPORT BITSANDBYTES SO USE CUDA 11.8 FOR GPU ACCELERATION, IF ANY ISSUE PERSISTS**"
      ],
      "metadata": {
        "id": "vWbOuIUVuFyW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model, tokenizer, device = setup_qwen_model()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "5259526d84a949c49876d3023f1a93fd",
            "5c7925792d36448f9b203f2a1f4e0029",
            "0ac0ef82c7784e4f9dbe34d631ed4572",
            "ecf0bd2b06c54eb6b7f3822e9320922a",
            "6b6e65925dd54c0f919605472ca63de6",
            "dbcecf58452d4602accb6e5e6ff091c6",
            "5435ed62749445748dbe751d2905f8db",
            "dfea50bdbb254c37b0dfee8e3502d8b3",
            "081cc0769ade4be9a862110d6d50ea91",
            "f10b461eef9a42cb8347d17a4348044d",
            "5bd0ffa967924ac09d813fff391fb4eb"
          ]
        },
        "outputId": "3fc7ecf8-c546-4450-8ede-2f4a3ae9cec0",
        "id": "Nw6emQnHuFyW"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5259526d84a949c49876d3023f1a93fd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:accelerate.big_modeling:Some parameters are on the meta device because they were offloaded to the disk and cpu.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "if __name__ == \"__main__\":\n",
        "    # --- Data Loading ---\n",
        "    repo_url = \"https://raw.githubusercontent.com/403errors/CancerCareAI/main/data\"\n",
        "    filenames = [\"1.json\", \"2.json\", \"3.json\"]\n",
        "    patient_data = load_data_from_github(repo_url, filenames)\n",
        "    if patient_data is None:\n",
        "        print(\"Data loading failed. Exiting.\")\n",
        "        exit()\n",
        "\n",
        "    # --- Patient Selection ---\n",
        "    print(\"Available patients:\")\n",
        "    patient_names = {}\n",
        "    for filename, records in patient_data.items():\n",
        "        first_record = records[0]\n",
        "        if \"docText\" in first_record:\n",
        "            name_match = re.search(r\"Patient Name:\\s*([^\\n]+)\", first_record[\"docText\"])\n",
        "            if name_match:\n",
        "                patient_name = name_match.group(1).strip()\n",
        "                patient_names[filename] = patient_name\n",
        "                print(f\"- {patient_name} ({filename})\")  # Display \"Lisa Bowman (1.json)\"\n",
        "            else:\n",
        "                print(f\"- {filename} (Could not extract patient name)\")\n",
        "                patient_names[filename] = None\n",
        "        else:\n",
        "            print(f\"- {filename} (Missing docText)\")\n",
        "            patient_names[filename] = None\n",
        "\n",
        "    selected_patient = input(\"Enter the full name of the patient you want to process ie. Lisa Bowman: \")\n",
        "    selected_file = None\n",
        "\n",
        "    # Find the file associated with the selected patient name.  Handle potential duplicates.\n",
        "    matching_files = [fname for fname, pname in patient_names.items() if pname == selected_patient]\n",
        "    if not matching_files:\n",
        "        print(f\"Error: No patient found with the name '{selected_patient}'.\")\n",
        "        exit()\n",
        "    elif len(matching_files) > 1:\n",
        "        print(\"Multiple files found for this patient. Please select one:\")\n",
        "        for i, fname in enumerate(matching_files):\n",
        "            print(f\"{i+1}. {fname}\")\n",
        "        choice = int(input(\"Enter the number of the file: \")) - 1 # User enters 1, 2..\n",
        "        selected_file = matching_files[choice]\n",
        "    else:\n",
        "        selected_file = matching_files[0]\n",
        "\n",
        "    selected_patient_data = {selected_file: patient_data[selected_file]}\n",
        "\n",
        "    # --- Mode Selection ---\n",
        "    print(\"\\nSelect a mode:\")\n",
        "    print(\"1. Query (Information Retrieval)\")\n",
        "    print(\"2. Medical Data Extraction\")\n",
        "    mode = input(\"Enter your choice (1 or 2): \")\n",
        "\n",
        "    if model is None and mode == '2':\n",
        "        print(\"Model is not available on CPU, Exiting.\")\n",
        "        exit()\n",
        "\n",
        "\n",
        "    if mode == \"1\":\n",
        "        # --- Query Mode (Task 1) - Sentence Level ---\n",
        "        query = input(\"Enter your query: \")\n",
        "        passages = create_passages(selected_patient_data)  # Create sentence-level passages\n",
        "        retrieved_passages = combined_retrieval(query, passages)\n",
        "\n",
        "        print(\"\\nTop Retrieved Sentences:\")  # Changed output label\n",
        "        count = 1\n",
        "        for passage, score in retrieved_passages[:5]:\n",
        "            print(f\"{count}. {passage['docText']}\")  # Display the sentence\n",
        "            count += 1\n",
        "\n",
        "    elif mode == \"2\":\n",
        "        # --- Medical Data Extraction Mode (Task 2) ---\n",
        "        aggregated_data = {\n",
        "            \"diagnosis_characteristics\": [],\n",
        "            \"cancer_related_medications\": []\n",
        "        }\n",
        "\n",
        "        # Iterate through DOCUMENTS (not sentences)\n",
        "        total_documents = len(selected_patient_data[selected_file])\n",
        "        print(f\"\\nExtracting data from {total_documents} documents...\")\n",
        "        document_number = 1\n",
        "        for record in selected_patient_data[selected_file]:\n",
        "            passage = {\n",
        "                \"docText\": record[\"docText\"],\n",
        "                \"docTitle\": record[\"docTitle\"],\n",
        "                \"docDate\": record[\"docDate\"],\n",
        "                \"patient_file\": selected_file\n",
        "            }\n",
        "            extracted_data = extract_information(model, tokenizer, device, passage)\n",
        "\n",
        "            # Merge the extracted data into the aggregated data\n",
        "            aggregated_data = merge_extractions(aggregated_data, extracted_data)\n",
        "\n",
        "            print(f\"Processed record: {document_number}/{total_documents}\")\n",
        "            document_number += 1\n",
        "\n",
        "\n",
        "        if aggregated_data:\n",
        "            print(\"\\nExtracted Data (JSON):\")\n",
        "            print(json.dumps(aggregated_data, indent=2))\n",
        "        else:\n",
        "            print(\"No data extracted.\")\n",
        "\n",
        "    else:\n",
        "        print(\"Invalid mode selected.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 835,
          "referenced_widgets": [
            "f0c19510c4a34610aaa909717dcbd802",
            "718044bbf2bc4ea18abcabce890e1d43",
            "8438654f2fb8413ba14a4d2e92b5e9cd",
            "e7ddd885e14947718dcf8aa93fc84422",
            "4d99405c3fd04573ae98d26cebb37613",
            "5bd483524992453d946a1bcb08fcdc42",
            "ecb019b2345741fda156bb775e6f130a",
            "9f0138a02b2d48f1bbed873f141fce6e",
            "7f61617737d4495282fd400b8432fac4",
            "e0ebeb32edee4ecc96bace1c2ff2538c",
            "31b49023f2dc427f8fc124b7fd75f2d4",
            "047b745ac0d4402cbfa08009aaa6733e",
            "1a421ae173bb4a16820a82fe3dda619f",
            "686bb31d75bb4d5689ce2c4652be0f01",
            "183e96bf39ab48e4a68e1bf9d509a204",
            "b52204d2c95b47869c0dd4a708e2f67c",
            "1c2ebf339284485f9b718f4b5bcd1706",
            "d59abedcf65f4624a92442eaff198220",
            "a9bc3e91cab949bb8705ca626b0fe9d0",
            "e8e1fb6b190941de9a8d5bd1e59efe37",
            "2e4f761abedf42af891f1a678df4cc61",
            "f337ceb31edd40aa8acf4aa6d1cced8f",
            "4b049e62269643e0b5984ed1b2fdb7a4",
            "eeccb5583a97477c96bf49cc5f6af08a",
            "c3c3d920398948fd87c345e12bb2bfbe",
            "033f70802d4f42429b3a02d81ce27907",
            "615deb345d9d4c979e52512b6d9f441e",
            "b3a6e5b6d3b24039bcb0a076e4925aac",
            "b8c39aa87f8345fd89a840d9b4f5e9cc",
            "0173f18df3df4576ac06711eba2522c5",
            "c752f512980641029308a4aebef77dd5",
            "1bbba044e4284cdf8253480c2febab34",
            "4b9aed27781944d8b410fc9fc12337a5",
            "68ff5e32337d4042a75a4b5d336cfc3e",
            "8e9afeec5ae640f4869793289e21990f",
            "ec2dd7618ae9412cb82c631e7f697a0a",
            "9261de2268504afa9773bc335185ef8e",
            "cd161bc46e664c8297d56ae4c82e84b4",
            "a2d1b9ede216413fbc0b06bf8d535d87",
            "cc08457e99bb4d4c9765118ad32b4432",
            "eedb0003b5a44ed8a3492b9b9767f7aa",
            "1ad5e3e543834736ac916448dbf7942e",
            "53803673617146509e78d50103be08b7",
            "91f76d8c325e48a693a00f9ce63e042b",
            "a177c8207f474619a259cafcec9f7092",
            "33b7117829e2471ab389a4ac580f3dfe",
            "c5f70e46a2614e59ba27418b6a962d72",
            "556e0440d43a4644b56a5f8f117c1835",
            "872d83b987864c3d898e7d2c140e82fa",
            "6a8c8360cfa648e6bf841eadf8c945b9",
            "fb5d7764afdd43cea51bd9224d636e70",
            "8546810d75ad428c990a4bbb7ef50299",
            "f5dc325e14c34f0a9d94fa9fd1d92395",
            "26b5138e295744bf846857fd67aaf00e",
            "4b7098582dd54204a3347d3ffa33d8c2",
            "0f63fe78c65846f48a199b5383f58c0a",
            "773bd1939425431e8c6ffadbbc57962a",
            "e476c65a018641749ebef7fc084519c6",
            "29414cdcaa754cdcac22746d07977c1c",
            "faa73ab520dd41ec9f46ef0b0d8998a2",
            "f97bb59449e2410295c8d3403f71584b",
            "f3fb2740b6044bfcae41736a2192d267",
            "9ad734ec78744bfba2255b0978aa3182",
            "039e0bf28a68471da66171f629c1cfc5",
            "69c2c320e96540208581efd528ed1365",
            "04800d99933f4fa3b9136caebeea6e6c",
            "df7938cc773d46c3a034fddf2a3cf6db",
            "f985fe0492ca4a4ab49114f17aa97856",
            "826066ead988457ca79650e02b1b58c8",
            "d1e084a513d548cea600df056e288af1",
            "f57a4a3eab7f4c44ac775a8cb641a245",
            "312fa22f8d484864ba03da29200a86a7",
            "443b3c3d645a464fbd3a79297ff798b0",
            "58005c022f0c4036b0ddbb410451a3e5",
            "0c46bb5dec0b4e34872c189eb265347b",
            "f7eddb690676495a911c152ec04aa9d0",
            "f67de61f900f4334a64eaa688e7810d0",
            "cadf5e2a5f7e42dba04285ae15603818",
            "c224bbc8bd5c47ca95825b7553252841",
            "a3c55f64b1904c3d8dd76adb34cffa04",
            "24670bdd62444ac2823d633ff349a2c1",
            "df60e40742574e279ebe85ab7f70f1d5",
            "65c2968cd0384c939637d7e46110c078",
            "1b7698f7ba01497d99b59b6f101c9bdd",
            "f90f9efa6bfd46a787cbfa2592710ce1",
            "696d2f900cb942c7870afb254f4e78d5",
            "d2648085fb1c49d8b3a95a337fbd31c9",
            "e0372aa8e1da4f5b9c72e2074b04c2e1",
            "a83ffc75bd7c405abea31ec29d03e502",
            "2cbdcb1620ee47229def529cc998560f",
            "3c4aa00c0c874f728524604dfaba644b",
            "a18504145a464f35900da6c8a3ee6988",
            "065ccba57fba473fbc561f5ebee0e495",
            "6ebe6022574c4cc582dd2278fec08123",
            "371f6bb1d7554f6a8b91edf3bcbe29c2",
            "2e24f2a3adb2423c99b2bda00cd4b127",
            "c3d9481d68184f06954518d48f8358b9",
            "41f1814a17cf405da0927db2b209f6c8",
            "910b6c5a923e4c7b8be6629193153598",
            "3dc93b204fef47688e4e192c78b6fd57",
            "7549cd50d03d40c3b2cd776b1e00a453",
            "88ff9cc14dd54936ad46b45ac5767ecc",
            "00a7cca05a6647329905aaa2d30c0b1d",
            "0f0575a31a804f52bfa5369780dda10b",
            "e7838c8d9dd541f1a2f73e07dc4ff0e5",
            "529ab5d7ac6e40ef9d893963fd58c7c7",
            "7d98b2687e814f9090efef6daf300c5c",
            "9da58c0727ec43a2b01154a2ac64fcd0",
            "847d8ad666b143da87b4314b37a045ef",
            "2b9986d3c0f74fbf8669a9f432c6358b",
            "c7b2538370a3401f94d8c16e32183f79",
            "e1a35b1c4e0a4429a563999d80aa9fc0",
            "b9706c10f2cd4a09b50a67d041e9fbb0",
            "55108fab7765407ab1a147252d78e914",
            "ee8fc3ca6e96446795eeeb31e2d3a786",
            "01b3d95ff3ad4345a445eb4bf6dfcf5b",
            "2fc85a7050a34692869dadfd7a390a5f",
            "b62134174dcf48d6b0ba19281daeb334",
            "fd0bb2585dd843818e8e9f7cfdcda3b8",
            "eb01bc1619cb474e840dd79b3ed5a590",
            "f35a229e81f74d2c947e6accdf1a9b9f",
            "21eb7b8e8f8e493db211c181afe00157",
            "a64d1719126b49268ac7d5e6443a8ea7",
            "37f2482beb4d493d90c9e142765966ca",
            "cb066cbebb524dc68cf66ffcd326073b",
            "76ab1ce767d74fb4a724dd22975510c3",
            "1e7784ddfe2a4a16b0358ede341fda57",
            "28be114f1a934a049369fe573e8266c7",
            "3d25670a37664bb5a707c063252f2033",
            "d7b9a5bb6de74c7da2573ae1252c0f9e",
            "87a038e10f5744a4918522ec01076e4d",
            "9ee3ac9829f84d6f81ef2de57b9e7885",
            "6e5b1df168cf4c86bd25ea0bc10ba7e6",
            "0f3d8da3afc6420fae754720d04513bd",
            "ae49de46baac44afb68f595eb0ee5b19",
            "11c7eb7d17e34969a82868cd17333002",
            "c391a114d4484922856e3e7753ea8209",
            "03edf638d955429f8ea15eec7196d36a",
            "c21ea376aaab4d8580e4b83750f1e1a2",
            "8060d848c642458392d5ba348454ed6c",
            "1be4fa47f95a41ae8fca91ecd775ef68",
            "4e9541f4bb404f73b36a8525920a1ba1",
            "2ea18f2a44ff4088b9e301787a48d667",
            "b44e846240804d438c2d3c262aef77d9",
            "df217a01b6314bd4be9fd0f2c05cb728",
            "efad193509cb4f82ad74556b41a342ea",
            "27e09754ceb84ae0b292737e66d4508d",
            "36a73b80e2c84929be1c85370f32ef0d",
            "07ab8036a7514ddbb0c8ca10feada362",
            "b7bcc57eec094296a4d0577da3c55b49",
            "86509b6fcb474ec3a74b5373c47fa378",
            "483a6869629c49a89e1643bc55a829cf",
            "2a58efc57eef4e78a8564c5b149da7f1",
            "e59918aab07840a2b4ade5878e839d94",
            "0e40f453bf504ed997cd069b6bdf5f1d",
            "01a90406a1cc4eb8a41fefe72936f898",
            "1f39b43a3469423d921f70c68d0914ac",
            "53de88bb3f844b1a9c219f01684cccbb",
            "d98967e5db044e2caf030e423573606d",
            "126331a7e2d04d9c82afe77056dbd5f5",
            "f4354d92d04d48cfb5a871c1d5027872",
            "ff729e1cb5f94f4d944a115c4182de5c",
            "5d67bb76a6ab4b178ff9b2bc4ae6e2a8",
            "e70cc7c2e0e64989b2644c1163c276fd",
            "2c90b81ab4ad4581978acee8ab8cafff",
            "48b094a56b7d4699b82a7c35617ce225",
            "080f3ef294dc4a8ba5f4bc750c37c5d5",
            "64e179b510e64e6f8dee1bb6a7a3af6c",
            "9dafdf407c0d4a368950c56efc404cfe",
            "767bb0196c9b4a40afec965f2eda0950",
            "3b23c474865c4b92a0c13f0f1de5d232",
            "9667e0f08e4f4c6a95efa4896ce362f2",
            "e3ec0d072e874f82867787f270518702",
            "6feb504742ff481f925d8a897589b44b",
            "6c536eabac4640e78c98143fdcd68d48",
            "f35b7014b7744215b6fdd496ac66206f"
          ]
        },
        "outputId": "8e48f21e-1e23-4d77-af8c-7b144443d1aa",
        "id": "556Hb98FuFyW"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Available patients:\n",
            "- Lisa Bowman (1.json)\n",
            "- John Whitfield (2.json)\n",
            "- Paul Henderson (3.json)\n",
            "Enter the full name of the patient you want to process ie. Lisa Bowman: John Whitfield\n",
            "\n",
            "Select a mode:\n",
            "1. Query (Information Retrieval)\n",
            "2. Medical Data Extraction\n",
            "Enter your choice (1 or 2): 1\n",
            "Enter your query: Has the patient undergone chemotherapy?\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "modules.json:   0%|          | 0.00/349 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f0c19510c4a34610aaa909717dcbd802"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config_sentence_transformers.json:   0%|          | 0.00/116 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "047b745ac0d4402cbfa08009aaa6733e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "README.md:   0%|          | 0.00/10.7k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4b049e62269643e0b5984ed1b2fdb7a4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "sentence_bert_config.json:   0%|          | 0.00/53.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "68ff5e32337d4042a75a4b5d336cfc3e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/612 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a177c8207f474619a259cafcec9f7092"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/90.9M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0f63fe78c65846f48a199b5383f58c0a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/350 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "df7938cc773d46c3a034fddf2a3cf6db"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cadf5e2a5f7e42dba04285ae15603818"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a83ffc75bd7c405abea31ec29d03e502"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3dc93b204fef47688e4e192c78b6fd57"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "1_Pooling%2Fconfig.json:   0%|          | 0.00/190 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c7b2538370a3401f94d8c16e32183f79"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/794 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "21eb7b8e8f8e493db211c181afe00157"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/90.9M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6e5b1df168cf4c86bd25ea0bc10ba7e6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/316 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b44e846240804d438c2d3c262aef77d9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0e40f453bf504ed997cd069b6bdf5f1d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "48b094a56b7d4699b82a7c35617ce225"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Top Retrieved Sentences:\n",
            "1. The patient underwent similar pre-medications (dexamethasone, antiemetics, IV fluids).\n",
            "2. An oncology consultation is advised to discuss chemotherapy regimens and the role of radiation if needed.\n",
            "3. Appointment with Dr. Rebecca Olson (Medical Oncology) in 2 weeks to discuss adjuvant chemotherapy.\n",
            "4. - Await final pathology results, which will guide the necessity of adjuvant chemotherapy and/or radiation therapy.\n",
            "5. - Discuss the possibility of concurrent chemotherapy and radiation therapy if surgery is not an optimal route.\n"
          ]
        }
      ]
    }
  ]
}