notebooks/5_collaborative_&_hybrid.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import string\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/harman/venv/jupyter/lib/python3.8/site-packages/gensim/similarities/__init__.py:15: UserWarning: The gensim.similarities.levenshtein submodule is disabled, because the optional Levenshtein package <https://pypi.org/project/python-Levenshtein/> is unavailable. Install Levenhstein (e.g. `pip install python-Levenshtein`) to suppress this warning.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "from gensim import corpora, models, similarities\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'lightfm'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-b5034edda520>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mlightfm\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLightFM\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'lightfm'"
     ]
    }
   ],
   "source": [
    "from lightfm import LightFM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy\n",
    "from scipy.spatial.distance import cdist\n",
    "from scipy.sparse import csr_matrix\n",
    "from scipy.sparse.linalg import svds\n",
    "from scipy.linalg import svd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import silhouette_score, silhouette_samples\n",
    "from sklearn.metrics.pairwise import cosine_similarity, linear_kernel\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reading data sets\n",
    "\n",
    "1. rated_articles - Contains user_interests with ratings\n",
    "2. news_articles - Contains raw articles without user data amalgamation"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## news_articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Article_id</th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Date</th>\n",
       "      <th>Category</th>\n",
       "      <th>URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Fire at Vaishno Devi shrine complex; cash coun...</td>\n",
       "      <td>No one was injured in the fire, which broke ou...</td>\n",
       "      <td>June 8, 2021 7:28:32 pm</td>\n",
       "      <td>India</td>\n",
       "      <td>https://indianexpress.com/article/india/vaishn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Had not gone to meet Nawaz Sharif, says Uddhav...</td>\n",
       "      <td>Uddhav Thackeray led a delegation of his cabin...</td>\n",
       "      <td>June 8, 2021 6:56:40 pm</td>\n",
       "      <td>India</td>\n",
       "      <td>https://indianexpress.com/article/india/had-no...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Corruption case: Former Haryana I-T deputy com...</td>\n",
       "      <td>It was in 2016 that the CBI had arrested Nitin...</td>\n",
       "      <td>June 8, 2021  6:25:24 pm</td>\n",
       "      <td>India</td>\n",
       "      <td>https://indianexpress.com/article/india/corrup...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Kannur MP K Sudhakaran appointed chief of Cong...</td>\n",
       "      <td>Sudhakaran will replace Ramachandran who had a...</td>\n",
       "      <td>June 8, 2021  5:04:40 pm</td>\n",
       "      <td>India</td>\n",
       "      <td>https://indianexpress.com/article/india/sudhak...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Kerala girl of Class 5 writes to CJI, lauds SC...</td>\n",
       "      <td>Chief Justice N V Ramana responded to the Clas...</td>\n",
       "      <td>June 8, 2021  4:43:10 pm</td>\n",
       "      <td>India</td>\n",
       "      <td>https://indianexpress.com/article/india/kerala...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Article_id                                              Title  \\\n",
       "0           0  Fire at Vaishno Devi shrine complex; cash coun...   \n",
       "1           1  Had not gone to meet Nawaz Sharif, says Uddhav...   \n",
       "2           2  Corruption case: Former Haryana I-T deputy com...   \n",
       "3           3  Kannur MP K Sudhakaran appointed chief of Cong...   \n",
       "4           4  Kerala girl of Class 5 writes to CJI, lauds SC...   \n",
       "\n",
       "                                         Description  \\\n",
       "0  No one was injured in the fire, which broke ou...   \n",
       "1  Uddhav Thackeray led a delegation of his cabin...   \n",
       "2  It was in 2016 that the CBI had arrested Nitin...   \n",
       "3  Sudhakaran will replace Ramachandran who had a...   \n",
       "4  Chief Justice N V Ramana responded to the Clas...   \n",
       "\n",
       "                       Date Category  \\\n",
       "0   June 8, 2021 7:28:32 pm    India   \n",
       "1   June 8, 2021 6:56:40 pm    India   \n",
       "2  June 8, 2021  6:25:24 pm    India   \n",
       "3  June 8, 2021  5:04:40 pm    India   \n",
       "4  June 8, 2021  4:43:10 pm    India   \n",
       "\n",
       "                                                 URL  \n",
       "0  https://indianexpress.com/article/india/vaishn...  \n",
       "1  https://indianexpress.com/article/india/had-no...  \n",
       "2  https://indianexpress.com/article/india/corrup...  \n",
       "3  https://indianexpress.com/article/india/sudhak...  \n",
       "4  https://indianexpress.com/article/india/kerala...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(r'/Users/harman/Desktop/EDA_NLP/Recommendation System/data/0_news_articles.csv')\n",
    "data.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collaborative Filtering\n",
    "\n",
    "**Need:** Ratings Matrix so I generated user profile with ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2250, 8)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Article_id</th>\n",
       "      <th>Title</th>\n",
       "      <th>UserId</th>\n",
       "      <th>SessionId</th>\n",
       "      <th>Article Rank</th>\n",
       "      <th>Time Spent (seconds)</th>\n",
       "      <th>Ratings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Fire at Vaishno Devi shrine complex; cash coun...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Had not gone to meet Nawaz Sharif, says Uddhav...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>53</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Corruption case: Former Haryana I-T deputy com...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Kannur MP K Sudhakaran appointed chief of Cong...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Kerala girl of Class 5 writes to CJI, lauds SC...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>27</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Article_id                                              Title  UserId  \\\n",
       "0           0  Fire at Vaishno Devi shrine complex; cash coun...       1   \n",
       "1           1  Had not gone to meet Nawaz Sharif, says Uddhav...       1   \n",
       "2           2  Corruption case: Former Haryana I-T deputy com...       1   \n",
       "3           3  Kannur MP K Sudhakaran appointed chief of Cong...       1   \n",
       "4           4  Kerala girl of Class 5 writes to CJI, lauds SC...       1   \n",
       "\n",
       "   SessionId  Article Rank  Time Spent (seconds)  Ratings  \n",
       "0          1             1                     0        3  \n",
       "1          1             2                    53        1  \n",
       "2          1             3                     0        2  \n",
       "3          1             4                     0        2  \n",
       "4          1             5                    27        3  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating = pd.read_csv(r'data/3_user_rated_articles.csv')\n",
    "print(rating.shape)\n",
    "rating.drop(columns= rating.columns[0], \n",
    "        axis=1, \n",
    "        inplace=True)\n",
    "rating.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Article_id</th>\n",
       "      <th>Title</th>\n",
       "      <th>UserId</th>\n",
       "      <th>SessionId</th>\n",
       "      <th>Article Rank</th>\n",
       "      <th>Time Spent (seconds)</th>\n",
       "      <th>Ratings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2245</th>\n",
       "      <td>2245</td>\n",
       "      <td>Malaika Arora shares yoga asanas for healthy, ...</td>\n",
       "      <td>2235</td>\n",
       "      <td>2230</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2246</th>\n",
       "      <td>2246</td>\n",
       "      <td>COVID-19 diet: Khichdi is a ‘safe’ option; oth...</td>\n",
       "      <td>2236</td>\n",
       "      <td>2231</td>\n",
       "      <td>7</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2247</th>\n",
       "      <td>2247</td>\n",
       "      <td>‘Keep listening. The world wants to hear your ...</td>\n",
       "      <td>2237</td>\n",
       "      <td>2232</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2248</th>\n",
       "      <td>2248</td>\n",
       "      <td>Forget cold drinks, switch to refreshing bael ...</td>\n",
       "      <td>2238</td>\n",
       "      <td>2233</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2249</th>\n",
       "      <td>2249</td>\n",
       "      <td>‘Love wins’: Rita Wilson and Tom Hanks celebra...</td>\n",
       "      <td>2239</td>\n",
       "      <td>2234</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Article_id                                              Title  UserId  \\\n",
       "2245        2245  Malaika Arora shares yoga asanas for healthy, ...    2235   \n",
       "2246        2246  COVID-19 diet: Khichdi is a ‘safe’ option; oth...    2236   \n",
       "2247        2247  ‘Keep listening. The world wants to hear your ...    2237   \n",
       "2248        2248  Forget cold drinks, switch to refreshing bael ...    2238   \n",
       "2249        2249  ‘Love wins’: Rita Wilson and Tom Hanks celebra...    2239   \n",
       "\n",
       "      SessionId  Article Rank  Time Spent (seconds)  Ratings  \n",
       "2245       2230             6                     0        4  \n",
       "2246       2231             7                    14        3  \n",
       "2247       2232             8                     0        1  \n",
       "2248       2233             9                     0        1  \n",
       "2249       2234            10                     0        5  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of users:  2239\n",
      "Number of articles:  2250\n"
     ]
    }
   ],
   "source": [
    "n_users = int(rating.UserId.nunique())\n",
    "n_article = int(rating.Article_id.nunique())\n",
    "print(\"Number of users: \" , n_users)\n",
    "print(\"Number of articles: \", n_article)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Article_id</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>2240</th>\n",
       "      <th>2241</th>\n",
       "      <th>2242</th>\n",
       "      <th>2243</th>\n",
       "      <th>2244</th>\n",
       "      <th>2245</th>\n",
       "      <th>2246</th>\n",
       "      <th>2247</th>\n",
       "      <th>2248</th>\n",
       "      <th>2249</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UserId</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2250 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Article_id  0     1     2     3     4     5     6     7     8     9     ...  \\\n",
       "UserId                                                                  ...   \n",
       "1            3.0   1.0   2.0   2.0   3.0   2.0   5.0   5.0   3.0   2.0  ...   \n",
       "2            NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "3            NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "4            NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "5            NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "\n",
       "Article_id  2240  2241  2242  2243  2244  2245  2246  2247  2248  2249  \n",
       "UserId                                                                  \n",
       "1            NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2            NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "3            NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "4            NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "5            NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[5 rows x 2250 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_pivot = rating.pivot_table(index = 'UserId', columns = 'Article_id', values = 'Ratings')\n",
    "user_pivot.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2239, 2250)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_pivot.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3., 1., 2., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 1., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 1., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 5.]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_pivot = user_pivot.fillna(0)\n",
    "user_pivot_matrix = user_pivot.values\n",
    "user_pivot_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_item_pivot_sparse = csr_matrix(user_pivot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 150)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_factors = 150\n",
    "U, sigma, Vt = svds(user_pivot_matrix, k = n_factors)\n",
    "\n",
    "sigma = np.diag(sigma)\n",
    "sigma.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.65056069, 0.30112139, 0.47584104, ..., 0.12640174, 0.12640174,\n",
       "        0.12640174],\n",
       "       [0.12640174, 0.12640174, 0.12640174, ..., 0.12640174, 0.12640174,\n",
       "        0.12640174],\n",
       "       [0.12640174, 0.12640174, 0.12640174, ..., 0.12640174, 0.12640174,\n",
       "        0.12585864],\n",
       "       ...,\n",
       "       [0.12640174, 0.12640174, 0.12640174, ..., 0.12640174, 0.12640174,\n",
       "        0.12640174],\n",
       "       [0.12640174, 0.12640174, 0.12640174, ..., 0.12640174, 0.12640174,\n",
       "        0.12640174],\n",
       "       [0.12640174, 0.12640174, 0.12640174, ..., 0.12640174, 0.12640174,\n",
       "        0.12650087]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_user_ratings = np.dot(np.dot(U, sigma), Vt) \n",
    "all_user_ratings_norm = (all_user_ratings - all_user_ratings.min()) / (all_user_ratings.max() - all_user_ratings.min())\n",
    "all_user_ratings_norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cf_preds_df = pd.DataFrame(all_user_ratings_norm, columns = user_pivot.columns).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Collaborative:\n",
    "    \n",
    "    name = \"Collaborative Filter\"\n",
    "    \n",
    "    def __init__(self, predictions, items = None):\n",
    "        self.predictions = predictions\n",
    "        self.items = items\n",
    "        \n",
    "    def get_model_name(self):\n",
    "        return self.name\n",
    "            \n",
    "    def recommend_items(self, user_id, items_ignore = [], topn = 10, verbose = False):\n",
    "        sorted_preds = self.predictions[user_id].sort_values(ascending = False).reset_index()\n",
    "\n",
    "        recommendations = sorted_preds[~sorted_preds['Article_id'].isin(items_ignore)].head(topn)\n",
    "\n",
    "        if verbose:\n",
    "            if self.items is None:\n",
    "                raise Exception('\"items\" required in verbose mode')\n",
    "\n",
    "            recommendations = recommendations.merge(self.items, how = 'left', left_on = 'Article_id', \n",
    "                                                    right_on = 'Article_id')[['Article_id', 'Title']]\n",
    "\n",
    "\n",
    "        return recommendations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Collaborative(cf_preds_df, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Article_id</th>\n",
       "      <th>Title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>51</td>\n",
       "      <td>BSY to BJP MLAs: Focus on Covid, refrain from ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>222</td>\n",
       "      <td>Serum Institute of India seeks DCGI’s nod to m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>319</td>\n",
       "      <td>Row over wedding party in UP village: many say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>371</td>\n",
       "      <td>Govt’s Year 8 begins with dented public trust,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2015</td>\n",
       "      <td>Why is the second dose of COVID-19 vaccine so ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1052</td>\n",
       "      <td>Champions League: Havertz sorry for F-bomb, Ev...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>742</td>\n",
       "      <td>Post COVID complication among children a new c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>203</td>\n",
       "      <td>Parents, elders biggest counsellors: Goa BJP o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1297</td>\n",
       "      <td>Shapovalov, Ruud advance to final at clay-cour...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>948</td>\n",
       "      <td>No pressure on me, time to enjoy the WTC final...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Article_id                                              Title\n",
       "0          51  BSY to BJP MLAs: Focus on Covid, refrain from ...\n",
       "1         222  Serum Institute of India seeks DCGI’s nod to m...\n",
       "2         319  Row over wedding party in UP village: many say...\n",
       "3         371  Govt’s Year 8 begins with dented public trust,...\n",
       "4        2015  Why is the second dose of COVID-19 vaccine so ...\n",
       "5        1052  Champions League: Havertz sorry for F-bomb, Ev...\n",
       "6         742  Post COVID complication among children a new c...\n",
       "7         203  Parents, elders biggest counsellors: Goa BJP o...\n",
       "8        1297  Shapovalov, Ruud advance to final at clay-cour...\n",
       "9         948  No pressure on me, time to enjoy the WTC final..."
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.recommend_items(user_id = 224, verbose = True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hybrid Recommendor System\n",
    "\n",
    "**Using:** LightRF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Article_id</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>2240</th>\n",
       "      <th>2241</th>\n",
       "      <th>2242</th>\n",
       "      <th>2243</th>\n",
       "      <th>2244</th>\n",
       "      <th>2245</th>\n",
       "      <th>2246</th>\n",
       "      <th>2247</th>\n",
       "      <th>2248</th>\n",
       "      <th>2249</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UserId</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2250 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Article_id  0     1     2     3     4     5     6     7     8     9     ...  \\\n",
       "UserId                                                                  ...   \n",
       "1            3.0   1.0   2.0   2.0   3.0   2.0   5.0   5.0   3.0   2.0  ...   \n",
       "2            0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   \n",
       "3            0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   \n",
       "4            0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   \n",
       "5            0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   \n",
       "\n",
       "Article_id  2240  2241  2242  2243  2244  2245  2246  2247  2248  2249  \n",
       "UserId                                                                  \n",
       "1            0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "2            0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "3            0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "4            0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "5            0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "\n",
       "[5 rows x 2250 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_pivot.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "u_id = list(user_pivot.index)\n",
    "u_dict = {}\n",
    "counter = 0\n",
    "\n",
    "for i in u_id:\n",
    "    u_dict[i] = counter\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2239"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(u_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<2239x2250 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 2250 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert to csr matrix\n",
    "u_interaction_csr = csr_matrix(user_pivot.values)\n",
    "u_interaction_csr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_dict ={}\n",
    "df = rating[['Article_id', 'Title']].sort_values('Article_id').reset_index()\n",
    "\n",
    "for i in range(df.shape[0]):\n",
    "    item_dict[(df.loc[i,'Article_id'])] = df.loc[i,'Title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2250"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(item_dict)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LightFM not working, so hopes low"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LightFM(loss = 'warp', random_state = 2016, learning_rate = 0.90, no_components = 150, user_alpha = 0.000005)\n",
    "model = model.fit(u_interaction_csr, epochs = 100, num_threads = 16, verbose = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Fire at Vaishno Devi shrine complex; cash counter damaged',\n",
       " 'Had not gone to meet Nawaz Sharif, says Uddhav Thackeray as he plays down one-on-one meeting with PM Modi',\n",
       " 'Corruption case: Former Haryana I-T deputy commissioner gets 4 years in prison',\n",
       " 'Kannur MP K Sudhakaran appointed chief of Congress in Kerala',\n",
       " 'Kerala girl of Class 5 writes to CJI, lauds SC for saving lives in fight with Covid',\n",
       " 'Madhya Pradesh govt gets HC notice on communal clashes during fundraising for Ram temple',\n",
       " 'Uddhav Thackeray meets PM Modi; discusses Maratha quota issue, GST compensation',\n",
       " 'New Covid-19 vaccination guidelines out, allocation based on state population',\n",
       " 'Dantewada: 24-year-old tribal woman killed in ‘maoist encounter’; family claims it was staged, alleges rape',\n",
       " 'Fire at TMC MLA Madan Mitra’s residence in Kolkata']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title = list(data['Title'])\n",
    "title[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Total</th>\n",
       "      <th>Percent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Article_id</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Title</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Description</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Category</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Total  Percent\n",
       "Article_id       0      0.0\n",
       "Title            0      0.0\n",
       "Description      0      0.0\n",
       "Date             0      0.0\n",
       "Category         0      0.0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total = data.isnull().sum().sort_values(ascending = False)\n",
    "percent = (data.isnull().sum()/data.isnull().count()).sort_values(ascending = False)\n",
    "missing_data = pd.concat([total, percent], axis = 1, keys = ['Total', 'Percent'])\n",
    "missing_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}