Visualization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.manifold import TSNE\n",
    "from transformers import AutoTokenizer, AutoModel\n",
    "import plotly.graph_objs as go\n",
    "import torch\n",
    "\n",
    "import openai\n",
    "import time\n",
    "import logging\n",
    "import os\n",
    "\n",
    "import src.config as config\n",
    "\n",
    "logging.basicConfig(level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.manifold import TSNE\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "def visualize_embeddings_from_json(json_file):\n",
    "    with open(json_file, 'r') as file:\n",
    "        hadith_json = json.load(file)\n",
    "      \n",
    "    filtered_hadiths = []\n",
    "\n",
    "    for idx in range(len(hadith_json)):\n",
    "        hadith = hadith_json[idx]\n",
    "        hadith_embeddings = np.array(hadith['embeddings'])\n",
    "        if isinstance(hadith_embeddings, np.ndarray) :\n",
    "            if hadith_embeddings.shape == (1536,) :\n",
    "                filtered_hadiths.append(hadith)\n",
    "    \n",
    "    \n",
    "    # Extract embeddings from JSON and convert them to numpy arrays\n",
    "    embeddings = [np.array(item['embeddings']) for item in filtered_hadiths]\n",
    "    \n",
    "    # Reduce dimensionality of embeddings using t-SNE\n",
    "    perplexity = min(len(embeddings) - 1, 30)  # Use either 30 or n_samples - 1, whichever is smaller\n",
    "    tsne = TSNE(n_components=3, random_state=7)\n",
    "    embeddings_3d = tsne.fit_transform(np.array(embeddings))\n",
    "\n",
    "    # Create a dictionary to map each surah to a unique color\n",
    "    colors = {}\n",
    "    for hadith in pd.DataFrame(filtered_hadiths)['Chapter_Number'].unique():\n",
    "        colors[hadith] = np.random.choice(range(256), size=3)\n",
    "\n",
    "    # Create scatter plot\n",
    "    fig = go.Figure()\n",
    "    for hadith in pd.DataFrame(filtered_hadiths)['Chapter_Number'].unique():\n",
    "        data = pd.DataFrame(filtered_hadiths)[pd.DataFrame(filtered_hadiths)['Chapter_Number'] == hadith]\n",
    "        color = f'rgb({\",\".join(map(str, colors[hadith]))})'\n",
    "        fig.add_trace(go.Scatter3d(\n",
    "            x=embeddings_3d[data.index, 0],\n",
    "            y=embeddings_3d[data.index, 1],\n",
    "            z=embeddings_3d[data.index, 2],\n",
    "            text= data[\"Arabic_Hadith\"],\n",
    "            mode='markers',\n",
    "            marker=dict(\n",
    "                size=10,\n",
    "                color=color,\n",
    "                opacity=0.8,\n",
    "            ),\n",
    "            name=data[\"Chapter_Arabic\"].unique()[0]\n",
    "        ))\n",
    "\n",
    "    # Set plot layout\n",
    "    fig.update_layout(\n",
    "        title='Distribution of Ibn Maja Hadiths in 3D',\n",
    "        scene=dict(\n",
    "            xaxis_title='t-SNE Dimension 1',\n",
    "            yaxis_title='t-SNE Dimension 2',\n",
    "            zaxis_title='t-SNE Dimension 3'\n",
    "        )\n",
    "    )\n",
    "\n",
    "    # Export plot as HTML file\n",
    "    fig.write_html(f'{json_file[:-5]}.html')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "visualize_embeddings_from_json('./data/embeddings/ibnmaja_GPT_embeddings_v2.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}