Semantic Search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The Task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on a user query, return an existing question which most closely resembles the user's query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "from sklearn import preprocessing\n",
    "\n",
    "import spacy\n",
    "from spacy.lang.en import English\n",
    "import spacy\n",
    "EN = spacy.load('en_core_web_sm')\n",
    "\n",
    "from IPython.display import HTML\n",
    "import logging\n",
    "logging.getLogger('tensorflow').disabled = True #OPTIONAL - to disable outputs from Tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>post_corpus</th>\n",
       "      <th>question_content</th>\n",
       "      <th>question_url</th>\n",
       "      <th>tags</th>\n",
       "      <th>overall_scores_norm</th>\n",
       "      <th>sentiment_polarity</th>\n",
       "      <th>sentiment_subjectivity</th>\n",
       "      <th>processed_title</th>\n",
       "      <th>original_title</th>\n",
       "      <th>overall_scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>using match attribute python objects array nt ...</td>\n",
       "      <td>using match attribute python objects array nt ...</td>\n",
       "      <td>https://stackoverflow.com/questions/683</td>\n",
       "      <td>python|arrays|iteration</td>\n",
       "      <td>0.012227</td>\n",
       "      <td>0.163567</td>\n",
       "      <td>0.568209</td>\n",
       "      <td>using match attribute python objects array</td>\n",
       "      <td>Using 'in' to match an attribute of Python obj...</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>specifying mysql enum django model go specifyi...</td>\n",
       "      <td>specifying mysql enum django model go specifyi...</td>\n",
       "      <td>https://stackoverflow.com/questions/21454</td>\n",
       "      <td>python|mysql|django|django-models|enums</td>\n",
       "      <td>0.029648</td>\n",
       "      <td>0.035299</td>\n",
       "      <td>0.536159</td>\n",
       "      <td>specifying mysql enum django model</td>\n",
       "      <td>Specifying a mySQL ENUM in a Django model</td>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unicode vs utf8 confusion python django stumbl...</td>\n",
       "      <td>unicode vs utf8 confusion python django stumbl...</td>\n",
       "      <td>https://stackoverflow.com/questions/22149</td>\n",
       "      <td>python|django|unicode</td>\n",
       "      <td>0.007405</td>\n",
       "      <td>0.082857</td>\n",
       "      <td>0.403250</td>\n",
       "      <td>unicode vs utf8 confusion python django</td>\n",
       "      <td>Unicode vs UTF-8 confusion in Python / Django?</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cleanest fastest server setup django deploy me...</td>\n",
       "      <td>cleanest fastest server setup django deploy me...</td>\n",
       "      <td>https://stackoverflow.com/questions/26025</td>\n",
       "      <td>python|django|apache|hosting</td>\n",
       "      <td>0.008494</td>\n",
       "      <td>0.205105</td>\n",
       "      <td>0.483688</td>\n",
       "      <td>cleanest fastest server setup django</td>\n",
       "      <td>Cleanest &amp; Fastest server setup for Django</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>always including user django template context ...</td>\n",
       "      <td>always including user django template context ...</td>\n",
       "      <td>https://stackoverflow.com/questions/41547</td>\n",
       "      <td>python|django|authentication|session|cookies</td>\n",
       "      <td>0.015027</td>\n",
       "      <td>0.052020</td>\n",
       "      <td>0.628824</td>\n",
       "      <td>always including user django template context</td>\n",
       "      <td>Always including the user in the django templa...</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>generate urls django django template language ...</td>\n",
       "      <td>generate urls django django template language ...</td>\n",
       "      <td>https://stackoverflow.com/questions/43290</td>\n",
       "      <td>python|django|url|django-urls</td>\n",
       "      <td>0.006472</td>\n",
       "      <td>0.080000</td>\n",
       "      <td>0.468333</td>\n",
       "      <td>generate urls django</td>\n",
       "      <td>How to generate urls in django</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>project design fs layout large django projects...</td>\n",
       "      <td>project design fs layout large django projects...</td>\n",
       "      <td>https://stackoverflow.com/questions/44135</td>\n",
       "      <td>python|django</td>\n",
       "      <td>0.004916</td>\n",
       "      <td>0.124153</td>\n",
       "      <td>0.470832</td>\n",
       "      <td>project design fs layout large django projects</td>\n",
       "      <td>Project design / FS layout for large django pr...</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>would make comma separated string list strings...</td>\n",
       "      <td>would make comma separated string list strings...</td>\n",
       "      <td>https://stackoverflow.com/questions/44778</td>\n",
       "      <td>python|list</td>\n",
       "      <td>0.139619</td>\n",
       "      <td>-0.014071</td>\n",
       "      <td>0.537851</td>\n",
       "      <td>would make comma separated string list strings</td>\n",
       "      <td>How would you make a comma-separated string fr...</td>\n",
       "      <td>903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>best django search app building django project...</td>\n",
       "      <td>best django search app building django project...</td>\n",
       "      <td>https://stackoverflow.com/questions/55056</td>\n",
       "      <td>python|django|search|search-engine</td>\n",
       "      <td>0.026226</td>\n",
       "      <td>0.181002</td>\n",
       "      <td>0.492883</td>\n",
       "      <td>best django search app</td>\n",
       "      <td>What's the best Django search app?</td>\n",
       "      <td>174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ignore case python strings easiest way compare...</td>\n",
       "      <td>ignore case python strings easiest way compare...</td>\n",
       "      <td>https://stackoverflow.com/questions/62567</td>\n",
       "      <td>python|string|case-insensitive</td>\n",
       "      <td>0.013160</td>\n",
       "      <td>0.075911</td>\n",
       "      <td>0.541590</td>\n",
       "      <td>ignore case python strings</td>\n",
       "      <td>Ignore case in Python strings</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>create new app startapp django googled around ...</td>\n",
       "      <td>create new app startapp django googled around ...</td>\n",
       "      <td>https://stackoverflow.com/questions/64237</td>\n",
       "      <td>python|django</td>\n",
       "      <td>0.012849</td>\n",
       "      <td>0.201130</td>\n",
       "      <td>0.437860</td>\n",
       "      <td>create new app startapp django</td>\n",
       "      <td>When to create a new app (with startapp) in Dj...</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>caching compiled regex objects python time pyt...</td>\n",
       "      <td>caching compiled regex objects python time pyt...</td>\n",
       "      <td>https://stackoverflow.com/questions/65266</td>\n",
       "      <td>python|regex|caching</td>\n",
       "      <td>0.001339</td>\n",
       "      <td>0.128704</td>\n",
       "      <td>0.576876</td>\n",
       "      <td>caching compiled regex objects python</td>\n",
       "      <td>Caching compiled regex objects in Python?</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>serving dynamically generated zip archives dja...</td>\n",
       "      <td>serving dynamically generated zip archives dja...</td>\n",
       "      <td>https://stackoverflow.com/questions/67454</td>\n",
       "      <td>python|django</td>\n",
       "      <td>0.017516</td>\n",
       "      <td>0.013797</td>\n",
       "      <td>0.402989</td>\n",
       "      <td>serving dynamically generated zip archives django</td>\n",
       "      <td>Serving dynamically generated ZIP archives in ...</td>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>python private methods actually private python...</td>\n",
       "      <td>python private methods actually private python...</td>\n",
       "      <td>https://stackoverflow.com/questions/70528</td>\n",
       "      <td>python|python-2.7|encapsulation|information-hi...</td>\n",
       "      <td>0.185971</td>\n",
       "      <td>0.098592</td>\n",
       "      <td>0.495546</td>\n",
       "      <td>python private methods actually private</td>\n",
       "      <td>Why are Python's 'private' methods not actuall...</td>\n",
       "      <td>1201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>create triple join table django using django b...</td>\n",
       "      <td>create triple join table django using django b...</td>\n",
       "      <td>https://stackoverflow.com/questions/72945</td>\n",
       "      <td>python|django|model-view-controller|model</td>\n",
       "      <td>0.000872</td>\n",
       "      <td>-0.052500</td>\n",
       "      <td>0.377292</td>\n",
       "      <td>create triple join table django</td>\n",
       "      <td>How to create a triple-join table with Django</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>use form values unbound form web report uses d...</td>\n",
       "      <td>use form values unbound form web report uses d...</td>\n",
       "      <td>https://stackoverflow.com/questions/75621</td>\n",
       "      <td>python|django</td>\n",
       "      <td>0.000095</td>\n",
       "      <td>0.066515</td>\n",
       "      <td>0.388939</td>\n",
       "      <td>use form values unbound form</td>\n",
       "      <td>How to use form values from an unbound form</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>split string list string happens tokens want s...</td>\n",
       "      <td>split string list string</td>\n",
       "      <td>https://stackoverflow.com/questions/88613</td>\n",
       "      <td>python|string|list|split</td>\n",
       "      <td>0.021249</td>\n",
       "      <td>-0.156337</td>\n",
       "      <td>0.653670</td>\n",
       "      <td>split string list</td>\n",
       "      <td>How do I split a string into a list?</td>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>python fastest algorithm removing duplicates l...</td>\n",
       "      <td>python fastest algorithm removing duplicates l...</td>\n",
       "      <td>https://stackoverflow.com/questions/89178</td>\n",
       "      <td>python|algorithm|list</td>\n",
       "      <td>0.017671</td>\n",
       "      <td>0.090934</td>\n",
       "      <td>0.534763</td>\n",
       "      <td>python fastest algorithm removing duplicates l...</td>\n",
       "      <td>In Python, what is the fastest algorithm for r...</td>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>django fcgid fedora core 9 missing fedora core...</td>\n",
       "      <td>django fcgid fedora core 9 missing fedora core...</td>\n",
       "      <td>https://stackoverflow.com/questions/92373</td>\n",
       "      <td>python|django|apache2|fastcgi|fcgid</td>\n",
       "      <td>-0.000683</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>django fcgid fedora core 9 missing</td>\n",
       "      <td>Django + FCGID on Fedora Core 9 -- what am I m...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>stripping non printable characters string pyth...</td>\n",
       "      <td>stripping non printable characters string pyth...</td>\n",
       "      <td>https://stackoverflow.com/questions/92438</td>\n",
       "      <td>python|string|non-printable</td>\n",
       "      <td>0.024515</td>\n",
       "      <td>0.195652</td>\n",
       "      <td>0.570736</td>\n",
       "      <td>stripping non printable characters string python</td>\n",
       "      <td>Stripping non printable characters from a stri...</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>send head http request python 2 trying get hea...</td>\n",
       "      <td>send head http request python 2 trying get hea...</td>\n",
       "      <td>https://stackoverflow.com/questions/107405</td>\n",
       "      <td>python|python-2.7|http|http-headers|content-type</td>\n",
       "      <td>0.050180</td>\n",
       "      <td>-0.004035</td>\n",
       "      <td>0.397661</td>\n",
       "      <td>send head http request python 2</td>\n",
       "      <td>How do you send a HEAD HTTP request in Python 2?</td>\n",
       "      <td>328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>change width form elements created modelform d...</td>\n",
       "      <td>change width form elements created modelform d...</td>\n",
       "      <td>https://stackoverflow.com/questions/110378</td>\n",
       "      <td>python|html|django|django-forms|django-templates</td>\n",
       "      <td>0.018760</td>\n",
       "      <td>0.171875</td>\n",
       "      <td>0.355655</td>\n",
       "      <td>change width form elements created modelform d...</td>\n",
       "      <td>Change the width of form elements created with...</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>list words dictionary start user input would g...</td>\n",
       "      <td>list words dictionary start user input would g...</td>\n",
       "      <td>https://stackoverflow.com/questions/112532</td>\n",
       "      <td>python|list|dictionary</td>\n",
       "      <td>0.005539</td>\n",
       "      <td>0.132461</td>\n",
       "      <td>0.478018</td>\n",
       "      <td>list words dictionary start user input</td>\n",
       "      <td>List all words in a dictionary that start with...</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>python regex vs php regex competition instead ...</td>\n",
       "      <td>python regex vs php regex competition instead ...</td>\n",
       "      <td>https://stackoverflow.com/questions/118143</td>\n",
       "      <td>php|python|regex</td>\n",
       "      <td>0.001961</td>\n",
       "      <td>0.156141</td>\n",
       "      <td>0.578046</td>\n",
       "      <td>python regex vs php regex</td>\n",
       "      <td>Python Regex vs PHP Regex</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>join list string caveat along lines previous q...</td>\n",
       "      <td>join list string caveat along lines previous q...</td>\n",
       "      <td>https://stackoverflow.com/questions/118458</td>\n",
       "      <td>python|string|list|csv</td>\n",
       "      <td>0.000717</td>\n",
       "      <td>0.039394</td>\n",
       "      <td>0.299242</td>\n",
       "      <td>join list string caveat</td>\n",
       "      <td>How can I join a list into a string (caveat)?</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>translate regular expression idiom perl python...</td>\n",
       "      <td>translate regular expression idiom perl python...</td>\n",
       "      <td>https://stackoverflow.com/questions/122277</td>\n",
       "      <td>python|regex|perl</td>\n",
       "      <td>0.009894</td>\n",
       "      <td>0.133661</td>\n",
       "      <td>0.507972</td>\n",
       "      <td>translate regular expression idiom perl python</td>\n",
       "      <td>How do you translate this regular-expression i...</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>adding new element text substring say followin...</td>\n",
       "      <td>adding new element text substring say followin...</td>\n",
       "      <td>https://stackoverflow.com/questions/125102</td>\n",
       "      <td>javascript|jquery|python|beautifulsoup</td>\n",
       "      <td>-0.000217</td>\n",
       "      <td>0.284930</td>\n",
       "      <td>0.536869</td>\n",
       "      <td>adding new element text substring</td>\n",
       "      <td>Adding New Element to Text Substring</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>editing xml dictionary python trying generate ...</td>\n",
       "      <td>editing xml dictionary python trying generate ...</td>\n",
       "      <td>https://stackoverflow.com/questions/127606</td>\n",
       "      <td>python|xml|dictionary</td>\n",
       "      <td>0.005850</td>\n",
       "      <td>0.102444</td>\n",
       "      <td>0.561338</td>\n",
       "      <td>editing xml dictionary python</td>\n",
       "      <td>Editing XML as a dictionary in python?</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>best way store set data python list data follo...</td>\n",
       "      <td>best way store set data python list data follo...</td>\n",
       "      <td>https://stackoverflow.com/questions/128259</td>\n",
       "      <td>python|dictionary|data-structures|set</td>\n",
       "      <td>-0.000217</td>\n",
       "      <td>-0.018955</td>\n",
       "      <td>0.501044</td>\n",
       "      <td>best way store set data python</td>\n",
       "      <td>What is the best way to store set data in Python?</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>easy way populate slugfield charfield built wa...</td>\n",
       "      <td>easy way populate slugfield charfield built wa...</td>\n",
       "      <td>https://stackoverflow.com/questions/141487</td>\n",
       "      <td>python|django|slug</td>\n",
       "      <td>0.010361</td>\n",
       "      <td>-0.039847</td>\n",
       "      <td>0.529898</td>\n",
       "      <td>easy way populate slugfield charfield</td>\n",
       "      <td>Is there an easy way to populate SlugField fro...</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140849</th>\n",
       "      <td>expand possibility put offline post event tryi...</td>\n",
       "      <td>expand possibility put offline post event tryi...</td>\n",
       "      <td>https://stackoverflow.com/questions/56405377</td>\n",
       "      <td>python|django|django-models|django-templates</td>\n",
       "      <td>-0.000683</td>\n",
       "      <td>-0.100000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>expand possibility put offline post event</td>\n",
       "      <td>Expand the possibility to put offline a post a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140850</th>\n",
       "      <td>slicing 1 none statement mean came across code...</td>\n",
       "      <td>slicing 1 none statement mean came across code...</td>\n",
       "      <td>https://stackoverflow.com/questions/56405849</td>\n",
       "      <td>python|arrays|numpy|numpy-slicing</td>\n",
       "      <td>-0.000683</td>\n",
       "      <td>0.144596</td>\n",
       "      <td>0.391893</td>\n",
       "      <td>slicing 1 none statement mean</td>\n",
       "      <td>Slicing with :-1 and None - What does each of ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140851</th>\n",
       "      <td>django get first choice value tuple index gett...</td>\n",
       "      <td>django get first choice value tuple index gett...</td>\n",
       "      <td>https://stackoverflow.com/questions/56405910</td>\n",
       "      <td>python|django|django-models|django-templates</td>\n",
       "      <td>-0.000372</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>django get first choice value tuple</td>\n",
       "      <td>Django: Get first Choice Value from tuple</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140852</th>\n",
       "      <td>creating dataframe dictionary arbitrary length...</td>\n",
       "      <td>creating dataframe dictionary arbitrary length...</td>\n",
       "      <td>https://stackoverflow.com/questions/56406088</td>\n",
       "      <td>python|dataframe|dictionary</td>\n",
       "      <td>-0.000683</td>\n",
       "      <td>-0.475000</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>creating dataframe dictionary arbitrary length...</td>\n",
       "      <td>Creating a dataframe from dictionary with arbi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140853</th>\n",
       "      <td>another way print multiple list elements list ...</td>\n",
       "      <td>another way print multiple list elements list ...</td>\n",
       "      <td>https://stackoverflow.com/questions/56406527</td>\n",
       "      <td>python|list</td>\n",
       "      <td>-0.000839</td>\n",
       "      <td>0.037963</td>\n",
       "      <td>0.494312</td>\n",
       "      <td>another way print multiple list elements list ...</td>\n",
       "      <td>Is there another way to print multiple list el...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140854</th>\n",
       "      <td>multiply every matrix numpy array transpose ev...</td>\n",
       "      <td>multiply every matrix numpy array transpose ev...</td>\n",
       "      <td>https://stackoverflow.com/questions/56406881</td>\n",
       "      <td>python|loops|numpy</td>\n",
       "      <td>-0.000528</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.437500</td>\n",
       "      <td>multiply every matrix numpy array transpose ev...</td>\n",
       "      <td>Multiply every matrix of a numpy array with th...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140855</th>\n",
       "      <td>pythonic way handling python code handling nan...</td>\n",
       "      <td>pythonic way handling python code handling nan...</td>\n",
       "      <td>https://stackoverflow.com/questions/56407075</td>\n",
       "      <td>python-3.x|pandas|dataframe</td>\n",
       "      <td>-0.000683</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>pythonic way handling python code handling nan...</td>\n",
       "      <td>Pythonic way of handling a python code for han...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140856</th>\n",
       "      <td>django find pillow though installed django str...</td>\n",
       "      <td>django find pillow though installed django str...</td>\n",
       "      <td>https://stackoverflow.com/questions/56407216</td>\n",
       "      <td>python|django|docker|python-imaging-library|al...</td>\n",
       "      <td>-0.000683</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>django find pillow though installed</td>\n",
       "      <td>Django does not find Pillow though installed</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140857</th>\n",
       "      <td>setting textinput focus equal true makes texti...</td>\n",
       "      <td>setting textinput focus equal true makes texti...</td>\n",
       "      <td>https://stackoverflow.com/questions/56407412</td>\n",
       "      <td>python|python-3.x|kivy|kivy-language</td>\n",
       "      <td>-0.000683</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.562500</td>\n",
       "      <td>setting textinput focus equal true makes texti...</td>\n",
       "      <td>Why is it that setting TextInput focus equal t...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140858</th>\n",
       "      <td>display dictionary print statement way print c...</td>\n",
       "      <td>display dictionary print statement way print c...</td>\n",
       "      <td>https://stackoverflow.com/questions/56407797</td>\n",
       "      <td>python|python-3.x|dictionary</td>\n",
       "      <td>-0.000839</td>\n",
       "      <td>0.107143</td>\n",
       "      <td>0.628571</td>\n",
       "      <td>display dictionary print statement</td>\n",
       "      <td>Display dictionary in print statement</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140859</th>\n",
       "      <td>pygametransformscale work game surface writing...</td>\n",
       "      <td>pygametransformscale work game surface writing...</td>\n",
       "      <td>https://stackoverflow.com/questions/56407891</td>\n",
       "      <td>python|python-3.x|pygame|zoom</td>\n",
       "      <td>-0.000839</td>\n",
       "      <td>-0.271717</td>\n",
       "      <td>0.474369</td>\n",
       "      <td>pygametransformscale work game surface</td>\n",
       "      <td>pygame.transform.scale does not work on the \"g...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140860</th>\n",
       "      <td>return multi line dictionary currently refacto...</td>\n",
       "      <td>return multi line dictionary currently refacto...</td>\n",
       "      <td>https://stackoverflow.com/questions/56408152</td>\n",
       "      <td>python|dictionary|return|multiline</td>\n",
       "      <td>-0.000372</td>\n",
       "      <td>0.049167</td>\n",
       "      <td>0.350000</td>\n",
       "      <td>return multi line dictionary</td>\n",
       "      <td>Return a multi-line dictionary</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140861</th>\n",
       "      <td>attributeerror file object attribute _ committ...</td>\n",
       "      <td>attributeerror file object attribute _ committ...</td>\n",
       "      <td>https://stackoverflow.com/questions/56408714</td>\n",
       "      <td>python|django</td>\n",
       "      <td>-0.000839</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>attributeerror file object attribute _ committ...</td>\n",
       "      <td>AttributeError: 'file' object has no attribute...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140862</th>\n",
       "      <td>plotting scatterplots dataframe grid matplotli...</td>\n",
       "      <td>plotting scatterplots dataframe grid matplotli...</td>\n",
       "      <td>https://stackoverflow.com/questions/56408988</td>\n",
       "      <td>python|matplotlib|seaborn|scatter-plot</td>\n",
       "      <td>-0.000994</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>plotting scatterplots dataframe grid matplotlib</td>\n",
       "      <td>Plotting scatterplots from a dataframe on a gr...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140863</th>\n",
       "      <td>fix error valueerror follower follower object ...</td>\n",
       "      <td>fix error valueerror follower follower object ...</td>\n",
       "      <td>https://stackoverflow.com/questions/56409055</td>\n",
       "      <td>python|django|github</td>\n",
       "      <td>-0.000839</td>\n",
       "      <td>0.078571</td>\n",
       "      <td>0.615476</td>\n",
       "      <td>fix error valueerror follower follower object ...</td>\n",
       "      <td>How to fix error \"ValueError: &lt;Follower: Follo...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140864</th>\n",
       "      <td>displaying large image next 3 smaller images s...</td>\n",
       "      <td>displaying large image next 3 smaller images s...</td>\n",
       "      <td>https://stackoverflow.com/questions/56409225</td>\n",
       "      <td>python|python-3.x|pyqt|pyqt5</td>\n",
       "      <td>-0.000683</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>displaying large image next 3 smaller images</td>\n",
       "      <td>Displaying a large image next to 3 smaller images</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140865</th>\n",
       "      <td>get specific word fron strcontains pandas data...</td>\n",
       "      <td>get specific word fron strcontains pandas data...</td>\n",
       "      <td>https://stackoverflow.com/questions/56409663</td>\n",
       "      <td>python-3.x|pandas</td>\n",
       "      <td>-0.000528</td>\n",
       "      <td>0.184000</td>\n",
       "      <td>0.630000</td>\n",
       "      <td>get specific word fron strcontains</td>\n",
       "      <td>How to get the specific word fron str.contains</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140866</th>\n",
       "      <td>installation tensorflow aarch64 error could fi...</td>\n",
       "      <td>installation tensorflow aarch64 error could fi...</td>\n",
       "      <td>https://stackoverflow.com/questions/56410003</td>\n",
       "      <td>python|tensorflow|pip|arm64</td>\n",
       "      <td>-0.000839</td>\n",
       "      <td>0.351587</td>\n",
       "      <td>0.702381</td>\n",
       "      <td>installation tensorflow aarch64 error could fi...</td>\n",
       "      <td>Installation of Tensorflow on aarch64: Error: ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140867</th>\n",
       "      <td>need filter data rows data past certain month ...</td>\n",
       "      <td>need filter data rows data past certain month ...</td>\n",
       "      <td>https://stackoverflow.com/questions/56410155</td>\n",
       "      <td>python|pandas</td>\n",
       "      <td>-0.000217</td>\n",
       "      <td>-0.146667</td>\n",
       "      <td>0.563810</td>\n",
       "      <td>need filter data rows data past certain month</td>\n",
       "      <td>Need to filter data rows where the data is pas...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140868</th>\n",
       "      <td>find words containing specific letters trying ...</td>\n",
       "      <td>find words containing specific letters trying ...</td>\n",
       "      <td>https://stackoverflow.com/questions/56410399</td>\n",
       "      <td>python|regex|string|regex-negation|python-regex</td>\n",
       "      <td>-0.000372</td>\n",
       "      <td>-0.114955</td>\n",
       "      <td>0.466861</td>\n",
       "      <td>find words containing specific letters</td>\n",
       "      <td>How to Find Words Not Containing Specific Lett...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140869</th>\n",
       "      <td>focus lost event dispatcher kivy textinput way...</td>\n",
       "      <td>focus lost event dispatcher kivy textinput way...</td>\n",
       "      <td>https://stackoverflow.com/questions/56410573</td>\n",
       "      <td>python|python-3.x|kivy|kivy-language</td>\n",
       "      <td>-0.000839</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>focus lost event dispatcher kivy textinput</td>\n",
       "      <td>Is there a focus lost event dispatcher for a k...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140870</th>\n",
       "      <td>shift maximum value difficult formulate questi...</td>\n",
       "      <td>shift maximum value difficult formulate questi...</td>\n",
       "      <td>https://stackoverflow.com/questions/56410654</td>\n",
       "      <td>python|pandas|dataframe</td>\n",
       "      <td>-0.000839</td>\n",
       "      <td>-0.562500</td>\n",
       "      <td>0.775000</td>\n",
       "      <td>shift maximum value</td>\n",
       "      <td>Shift the maximum value</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140871</th>\n",
       "      <td>checking two dictionaries key val pairs values...</td>\n",
       "      <td>checking two dictionaries key val pairs values...</td>\n",
       "      <td>https://stackoverflow.com/questions/56410679</td>\n",
       "      <td>python|numpy</td>\n",
       "      <td>-0.000994</td>\n",
       "      <td>-0.182500</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>checking two dictionaries key val pairs values...</td>\n",
       "      <td>Checking if two dictionaries of key-val pairs ...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140872</th>\n",
       "      <td>unnest explode multiple list 20 background fix...</td>\n",
       "      <td>unnest explode multiple list 20 background</td>\n",
       "      <td>https://stackoverflow.com/questions/56410811</td>\n",
       "      <td>python-3.x|pandas|dataframe</td>\n",
       "      <td>-0.000839</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>unnest explode multiple list 20</td>\n",
       "      <td>unnest (explode) multiple list 2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140873</th>\n",
       "      <td>limit users integer input want user input inte...</td>\n",
       "      <td>limit users integer input want user input inte...</td>\n",
       "      <td>https://stackoverflow.com/questions/56411224</td>\n",
       "      <td>python|python-3.x</td>\n",
       "      <td>-0.000372</td>\n",
       "      <td>-0.034615</td>\n",
       "      <td>0.603846</td>\n",
       "      <td>limit users integer input</td>\n",
       "      <td>How to limit a users integer input?</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140874</th>\n",
       "      <td>stop loop index value attempting use loop iter...</td>\n",
       "      <td>stop loop index value attempting use loop iter...</td>\n",
       "      <td>https://stackoverflow.com/questions/56411403</td>\n",
       "      <td>python|loops|selenium</td>\n",
       "      <td>-0.000061</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>stop loop index value</td>\n",
       "      <td>Stop a For Loop at an Index Value</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140875</th>\n",
       "      <td>grab value dictionary contains multiple keys d...</td>\n",
       "      <td>grab value dictionary contains multiple keys d...</td>\n",
       "      <td>https://stackoverflow.com/questions/56411413</td>\n",
       "      <td>python|dictionary</td>\n",
       "      <td>-0.000683</td>\n",
       "      <td>-0.155303</td>\n",
       "      <td>0.862879</td>\n",
       "      <td>grab value dictionary contains multiple keys</td>\n",
       "      <td>How to grab a value from a dictionary which co...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140876</th>\n",
       "      <td>python regex pattern always start end noticed ...</td>\n",
       "      <td>python regex pattern always start end noticed ...</td>\n",
       "      <td>https://stackoverflow.com/questions/56411530</td>\n",
       "      <td>python|regex|python-regex</td>\n",
       "      <td>-0.000217</td>\n",
       "      <td>0.246528</td>\n",
       "      <td>0.512500</td>\n",
       "      <td>python regex pattern always start end</td>\n",
       "      <td>In Python, should a regex pattern always start...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140877</th>\n",
       "      <td>use pandas filter unique test loop large dataf...</td>\n",
       "      <td>use pandas filter unique test loop large dataf...</td>\n",
       "      <td>https://stackoverflow.com/questions/56411591</td>\n",
       "      <td>python|python-3.x|pandas</td>\n",
       "      <td>-0.000839</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>use pandas filter unique test loop</td>\n",
       "      <td>How can I use a pandas filter and unique for t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140878</th>\n",
       "      <td>way visualize azimuthal average 3d cartesian s...</td>\n",
       "      <td>way visualize azimuthal average 3d cartesian s...</td>\n",
       "      <td>https://stackoverflow.com/questions/56411676</td>\n",
       "      <td>python|arrays|cartesian-coordinates</td>\n",
       "      <td>-0.000839</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.325000</td>\n",
       "      <td>way visualize azimuthal average 3d cartesian s...</td>\n",
       "      <td>Is there a way to visualize an azimuthal avera...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>140879 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              post_corpus  \\\n",
       "0       using match attribute python objects array nt ...   \n",
       "1       specifying mysql enum django model go specifyi...   \n",
       "2       unicode vs utf8 confusion python django stumbl...   \n",
       "3       cleanest fastest server setup django deploy me...   \n",
       "4       always including user django template context ...   \n",
       "5       generate urls django django template language ...   \n",
       "6       project design fs layout large django projects...   \n",
       "7       would make comma separated string list strings...   \n",
       "8       best django search app building django project...   \n",
       "9       ignore case python strings easiest way compare...   \n",
       "10      create new app startapp django googled around ...   \n",
       "11      caching compiled regex objects python time pyt...   \n",
       "12      serving dynamically generated zip archives dja...   \n",
       "13      python private methods actually private python...   \n",
       "14      create triple join table django using django b...   \n",
       "15      use form values unbound form web report uses d...   \n",
       "16      split string list string happens tokens want s...   \n",
       "17      python fastest algorithm removing duplicates l...   \n",
       "18      django fcgid fedora core 9 missing fedora core...   \n",
       "19      stripping non printable characters string pyth...   \n",
       "20      send head http request python 2 trying get hea...   \n",
       "21      change width form elements created modelform d...   \n",
       "22      list words dictionary start user input would g...   \n",
       "23      python regex vs php regex competition instead ...   \n",
       "24      join list string caveat along lines previous q...   \n",
       "25      translate regular expression idiom perl python...   \n",
       "26      adding new element text substring say followin...   \n",
       "27      editing xml dictionary python trying generate ...   \n",
       "28      best way store set data python list data follo...   \n",
       "29      easy way populate slugfield charfield built wa...   \n",
       "...                                                   ...   \n",
       "140849  expand possibility put offline post event tryi...   \n",
       "140850  slicing 1 none statement mean came across code...   \n",
       "140851  django get first choice value tuple index gett...   \n",
       "140852  creating dataframe dictionary arbitrary length...   \n",
       "140853  another way print multiple list elements list ...   \n",
       "140854  multiply every matrix numpy array transpose ev...   \n",
       "140855  pythonic way handling python code handling nan...   \n",
       "140856  django find pillow though installed django str...   \n",
       "140857  setting textinput focus equal true makes texti...   \n",
       "140858  display dictionary print statement way print c...   \n",
       "140859  pygametransformscale work game surface writing...   \n",
       "140860  return multi line dictionary currently refacto...   \n",
       "140861  attributeerror file object attribute _ committ...   \n",
       "140862  plotting scatterplots dataframe grid matplotli...   \n",
       "140863  fix error valueerror follower follower object ...   \n",
       "140864  displaying large image next 3 smaller images s...   \n",
       "140865  get specific word fron strcontains pandas data...   \n",
       "140866  installation tensorflow aarch64 error could fi...   \n",
       "140867  need filter data rows data past certain month ...   \n",
       "140868  find words containing specific letters trying ...   \n",
       "140869  focus lost event dispatcher kivy textinput way...   \n",
       "140870  shift maximum value difficult formulate questi...   \n",
       "140871  checking two dictionaries key val pairs values...   \n",
       "140872  unnest explode multiple list 20 background fix...   \n",
       "140873  limit users integer input want user input inte...   \n",
       "140874  stop loop index value attempting use loop iter...   \n",
       "140875  grab value dictionary contains multiple keys d...   \n",
       "140876  python regex pattern always start end noticed ...   \n",
       "140877  use pandas filter unique test loop large dataf...   \n",
       "140878  way visualize azimuthal average 3d cartesian s...   \n",
       "\n",
       "                                         question_content  \\\n",
       "0       using match attribute python objects array nt ...   \n",
       "1       specifying mysql enum django model go specifyi...   \n",
       "2       unicode vs utf8 confusion python django stumbl...   \n",
       "3       cleanest fastest server setup django deploy me...   \n",
       "4       always including user django template context ...   \n",
       "5       generate urls django django template language ...   \n",
       "6       project design fs layout large django projects...   \n",
       "7       would make comma separated string list strings...   \n",
       "8       best django search app building django project...   \n",
       "9       ignore case python strings easiest way compare...   \n",
       "10      create new app startapp django googled around ...   \n",
       "11      caching compiled regex objects python time pyt...   \n",
       "12      serving dynamically generated zip archives dja...   \n",
       "13      python private methods actually private python...   \n",
       "14      create triple join table django using django b...   \n",
       "15      use form values unbound form web report uses d...   \n",
       "16                               split string list string   \n",
       "17      python fastest algorithm removing duplicates l...   \n",
       "18      django fcgid fedora core 9 missing fedora core...   \n",
       "19      stripping non printable characters string pyth...   \n",
       "20      send head http request python 2 trying get hea...   \n",
       "21      change width form elements created modelform d...   \n",
       "22      list words dictionary start user input would g...   \n",
       "23      python regex vs php regex competition instead ...   \n",
       "24      join list string caveat along lines previous q...   \n",
       "25      translate regular expression idiom perl python...   \n",
       "26      adding new element text substring say followin...   \n",
       "27      editing xml dictionary python trying generate ...   \n",
       "28      best way store set data python list data follo...   \n",
       "29      easy way populate slugfield charfield built wa...   \n",
       "...                                                   ...   \n",
       "140849  expand possibility put offline post event tryi...   \n",
       "140850  slicing 1 none statement mean came across code...   \n",
       "140851  django get first choice value tuple index gett...   \n",
       "140852  creating dataframe dictionary arbitrary length...   \n",
       "140853  another way print multiple list elements list ...   \n",
       "140854  multiply every matrix numpy array transpose ev...   \n",
       "140855  pythonic way handling python code handling nan...   \n",
       "140856  django find pillow though installed django str...   \n",
       "140857  setting textinput focus equal true makes texti...   \n",
       "140858  display dictionary print statement way print c...   \n",
       "140859  pygametransformscale work game surface writing...   \n",
       "140860  return multi line dictionary currently refacto...   \n",
       "140861  attributeerror file object attribute _ committ...   \n",
       "140862  plotting scatterplots dataframe grid matplotli...   \n",
       "140863  fix error valueerror follower follower object ...   \n",
       "140864  displaying large image next 3 smaller images s...   \n",
       "140865  get specific word fron strcontains pandas data...   \n",
       "140866  installation tensorflow aarch64 error could fi...   \n",
       "140867  need filter data rows data past certain month ...   \n",
       "140868  find words containing specific letters trying ...   \n",
       "140869  focus lost event dispatcher kivy textinput way...   \n",
       "140870  shift maximum value difficult formulate questi...   \n",
       "140871  checking two dictionaries key val pairs values...   \n",
       "140872         unnest explode multiple list 20 background   \n",
       "140873  limit users integer input want user input inte...   \n",
       "140874  stop loop index value attempting use loop iter...   \n",
       "140875  grab value dictionary contains multiple keys d...   \n",
       "140876  python regex pattern always start end noticed ...   \n",
       "140877  use pandas filter unique test loop large dataf...   \n",
       "140878  way visualize azimuthal average 3d cartesian s...   \n",
       "\n",
       "                                        question_url  \\\n",
       "0            https://stackoverflow.com/questions/683   \n",
       "1          https://stackoverflow.com/questions/21454   \n",
       "2          https://stackoverflow.com/questions/22149   \n",
       "3          https://stackoverflow.com/questions/26025   \n",
       "4          https://stackoverflow.com/questions/41547   \n",
       "5          https://stackoverflow.com/questions/43290   \n",
       "6          https://stackoverflow.com/questions/44135   \n",
       "7          https://stackoverflow.com/questions/44778   \n",
       "8          https://stackoverflow.com/questions/55056   \n",
       "9          https://stackoverflow.com/questions/62567   \n",
       "10         https://stackoverflow.com/questions/64237   \n",
       "11         https://stackoverflow.com/questions/65266   \n",
       "12         https://stackoverflow.com/questions/67454   \n",
       "13         https://stackoverflow.com/questions/70528   \n",
       "14         https://stackoverflow.com/questions/72945   \n",
       "15         https://stackoverflow.com/questions/75621   \n",
       "16         https://stackoverflow.com/questions/88613   \n",
       "17         https://stackoverflow.com/questions/89178   \n",
       "18         https://stackoverflow.com/questions/92373   \n",
       "19         https://stackoverflow.com/questions/92438   \n",
       "20        https://stackoverflow.com/questions/107405   \n",
       "21        https://stackoverflow.com/questions/110378   \n",
       "22        https://stackoverflow.com/questions/112532   \n",
       "23        https://stackoverflow.com/questions/118143   \n",
       "24        https://stackoverflow.com/questions/118458   \n",
       "25        https://stackoverflow.com/questions/122277   \n",
       "26        https://stackoverflow.com/questions/125102   \n",
       "27        https://stackoverflow.com/questions/127606   \n",
       "28        https://stackoverflow.com/questions/128259   \n",
       "29        https://stackoverflow.com/questions/141487   \n",
       "...                                              ...   \n",
       "140849  https://stackoverflow.com/questions/56405377   \n",
       "140850  https://stackoverflow.com/questions/56405849   \n",
       "140851  https://stackoverflow.com/questions/56405910   \n",
       "140852  https://stackoverflow.com/questions/56406088   \n",
       "140853  https://stackoverflow.com/questions/56406527   \n",
       "140854  https://stackoverflow.com/questions/56406881   \n",
       "140855  https://stackoverflow.com/questions/56407075   \n",
       "140856  https://stackoverflow.com/questions/56407216   \n",
       "140857  https://stackoverflow.com/questions/56407412   \n",
       "140858  https://stackoverflow.com/questions/56407797   \n",
       "140859  https://stackoverflow.com/questions/56407891   \n",
       "140860  https://stackoverflow.com/questions/56408152   \n",
       "140861  https://stackoverflow.com/questions/56408714   \n",
       "140862  https://stackoverflow.com/questions/56408988   \n",
       "140863  https://stackoverflow.com/questions/56409055   \n",
       "140864  https://stackoverflow.com/questions/56409225   \n",
       "140865  https://stackoverflow.com/questions/56409663   \n",
       "140866  https://stackoverflow.com/questions/56410003   \n",
       "140867  https://stackoverflow.com/questions/56410155   \n",
       "140868  https://stackoverflow.com/questions/56410399   \n",
       "140869  https://stackoverflow.com/questions/56410573   \n",
       "140870  https://stackoverflow.com/questions/56410654   \n",
       "140871  https://stackoverflow.com/questions/56410679   \n",
       "140872  https://stackoverflow.com/questions/56410811   \n",
       "140873  https://stackoverflow.com/questions/56411224   \n",
       "140874  https://stackoverflow.com/questions/56411403   \n",
       "140875  https://stackoverflow.com/questions/56411413   \n",
       "140876  https://stackoverflow.com/questions/56411530   \n",
       "140877  https://stackoverflow.com/questions/56411591   \n",
       "140878  https://stackoverflow.com/questions/56411676   \n",
       "\n",
       "                                                     tags  \\\n",
       "0                                 python|arrays|iteration   \n",
       "1                 python|mysql|django|django-models|enums   \n",
       "2                                   python|django|unicode   \n",
       "3                            python|django|apache|hosting   \n",
       "4            python|django|authentication|session|cookies   \n",
       "5                           python|django|url|django-urls   \n",
       "6                                           python|django   \n",
       "7                                             python|list   \n",
       "8                      python|django|search|search-engine   \n",
       "9                          python|string|case-insensitive   \n",
       "10                                          python|django   \n",
       "11                                   python|regex|caching   \n",
       "12                                          python|django   \n",
       "13      python|python-2.7|encapsulation|information-hi...   \n",
       "14              python|django|model-view-controller|model   \n",
       "15                                          python|django   \n",
       "16                               python|string|list|split   \n",
       "17                                  python|algorithm|list   \n",
       "18                    python|django|apache2|fastcgi|fcgid   \n",
       "19                            python|string|non-printable   \n",
       "20       python|python-2.7|http|http-headers|content-type   \n",
       "21       python|html|django|django-forms|django-templates   \n",
       "22                                 python|list|dictionary   \n",
       "23                                       php|python|regex   \n",
       "24                                 python|string|list|csv   \n",
       "25                                      python|regex|perl   \n",
       "26                 javascript|jquery|python|beautifulsoup   \n",
       "27                                  python|xml|dictionary   \n",
       "28                  python|dictionary|data-structures|set   \n",
       "29                                     python|django|slug   \n",
       "...                                                   ...   \n",
       "140849       python|django|django-models|django-templates   \n",
       "140850                  python|arrays|numpy|numpy-slicing   \n",
       "140851       python|django|django-models|django-templates   \n",
       "140852                        python|dataframe|dictionary   \n",
       "140853                                        python|list   \n",
       "140854                                 python|loops|numpy   \n",
       "140855                        python-3.x|pandas|dataframe   \n",
       "140856  python|django|docker|python-imaging-library|al...   \n",
       "140857               python|python-3.x|kivy|kivy-language   \n",
       "140858                       python|python-3.x|dictionary   \n",
       "140859                      python|python-3.x|pygame|zoom   \n",
       "140860                 python|dictionary|return|multiline   \n",
       "140861                                      python|django   \n",
       "140862             python|matplotlib|seaborn|scatter-plot   \n",
       "140863                               python|django|github   \n",
       "140864                       python|python-3.x|pyqt|pyqt5   \n",
       "140865                                  python-3.x|pandas   \n",
       "140866                        python|tensorflow|pip|arm64   \n",
       "140867                                      python|pandas   \n",
       "140868    python|regex|string|regex-negation|python-regex   \n",
       "140869               python|python-3.x|kivy|kivy-language   \n",
       "140870                            python|pandas|dataframe   \n",
       "140871                                       python|numpy   \n",
       "140872                        python-3.x|pandas|dataframe   \n",
       "140873                                  python|python-3.x   \n",
       "140874                              python|loops|selenium   \n",
       "140875                                  python|dictionary   \n",
       "140876                          python|regex|python-regex   \n",
       "140877                           python|python-3.x|pandas   \n",
       "140878                python|arrays|cartesian-coordinates   \n",
       "\n",
       "        overall_scores_norm  sentiment_polarity  sentiment_subjectivity  \\\n",
       "0                  0.012227            0.163567                0.568209   \n",
       "1                  0.029648            0.035299                0.536159   \n",
       "2                  0.007405            0.082857                0.403250   \n",
       "3                  0.008494            0.205105                0.483688   \n",
       "4                  0.015027            0.052020                0.628824   \n",
       "5                  0.006472            0.080000                0.468333   \n",
       "6                  0.004916            0.124153                0.470832   \n",
       "7                  0.139619           -0.014071                0.537851   \n",
       "8                  0.026226            0.181002                0.492883   \n",
       "9                  0.013160            0.075911                0.541590   \n",
       "10                 0.012849            0.201130                0.437860   \n",
       "11                 0.001339            0.128704                0.576876   \n",
       "12                 0.017516            0.013797                0.402989   \n",
       "13                 0.185971            0.098592                0.495546   \n",
       "14                 0.000872           -0.052500                0.377292   \n",
       "15                 0.000095            0.066515                0.388939   \n",
       "16                 0.021249           -0.156337                0.653670   \n",
       "17                 0.017671            0.090934                0.534763   \n",
       "18                -0.000683            0.000000                0.500000   \n",
       "19                 0.024515            0.195652                0.570736   \n",
       "20                 0.050180           -0.004035                0.397661   \n",
       "21                 0.018760            0.171875                0.355655   \n",
       "22                 0.005539            0.132461                0.478018   \n",
       "23                 0.001961            0.156141                0.578046   \n",
       "24                 0.000717            0.039394                0.299242   \n",
       "25                 0.009894            0.133661                0.507972   \n",
       "26                -0.000217            0.284930                0.536869   \n",
       "27                 0.005850            0.102444                0.561338   \n",
       "28                -0.000217           -0.018955                0.501044   \n",
       "29                 0.010361           -0.039847                0.529898   \n",
       "...                     ...                 ...                     ...   \n",
       "140849            -0.000683           -0.100000                0.500000   \n",
       "140850            -0.000683            0.144596                0.391893   \n",
       "140851            -0.000372            0.000000                0.133333   \n",
       "140852            -0.000683           -0.475000                0.875000   \n",
       "140853            -0.000839            0.037963                0.494312   \n",
       "140854            -0.000528            0.050000                0.437500   \n",
       "140855            -0.000683            0.000000                0.000000   \n",
       "140856            -0.000683            0.000000                0.500000   \n",
       "140857            -0.000683            0.000000                0.562500   \n",
       "140858            -0.000839            0.107143                0.628571   \n",
       "140859            -0.000839           -0.271717                0.474369   \n",
       "140860            -0.000372            0.049167                0.350000   \n",
       "140861            -0.000839            0.000000                0.000000   \n",
       "140862            -0.000994            0.166667                0.833333   \n",
       "140863            -0.000839            0.078571                0.615476   \n",
       "140864            -0.000683            0.000000                0.050000   \n",
       "140865            -0.000528            0.184000                0.630000   \n",
       "140866            -0.000839            0.351587                0.702381   \n",
       "140867            -0.000217           -0.146667                0.563810   \n",
       "140868            -0.000372           -0.114955                0.466861   \n",
       "140869            -0.000839            0.000000                0.400000   \n",
       "140870            -0.000839           -0.562500                0.775000   \n",
       "140871            -0.000994           -0.182500                0.740000   \n",
       "140872            -0.000839            0.000000                0.000000   \n",
       "140873            -0.000372           -0.034615                0.603846   \n",
       "140874            -0.000061            0.375000                0.375000   \n",
       "140875            -0.000683           -0.155303                0.862879   \n",
       "140876            -0.000217            0.246528                0.512500   \n",
       "140877            -0.000839            0.700000                0.600000   \n",
       "140878            -0.000839            0.166667                0.325000   \n",
       "\n",
       "                                          processed_title  \\\n",
       "0              using match attribute python objects array   \n",
       "1                      specifying mysql enum django model   \n",
       "2                 unicode vs utf8 confusion python django   \n",
       "3                    cleanest fastest server setup django   \n",
       "4           always including user django template context   \n",
       "5                                    generate urls django   \n",
       "6          project design fs layout large django projects   \n",
       "7          would make comma separated string list strings   \n",
       "8                                  best django search app   \n",
       "9                              ignore case python strings   \n",
       "10                         create new app startapp django   \n",
       "11                  caching compiled regex objects python   \n",
       "12      serving dynamically generated zip archives django   \n",
       "13                python private methods actually private   \n",
       "14                        create triple join table django   \n",
       "15                           use form values unbound form   \n",
       "16                                      split string list   \n",
       "17      python fastest algorithm removing duplicates l...   \n",
       "18                     django fcgid fedora core 9 missing   \n",
       "19       stripping non printable characters string python   \n",
       "20                        send head http request python 2   \n",
       "21      change width form elements created modelform d...   \n",
       "22                 list words dictionary start user input   \n",
       "23                              python regex vs php regex   \n",
       "24                                join list string caveat   \n",
       "25         translate regular expression idiom perl python   \n",
       "26                      adding new element text substring   \n",
       "27                          editing xml dictionary python   \n",
       "28                         best way store set data python   \n",
       "29                  easy way populate slugfield charfield   \n",
       "...                                                   ...   \n",
       "140849          expand possibility put offline post event   \n",
       "140850                      slicing 1 none statement mean   \n",
       "140851                django get first choice value tuple   \n",
       "140852  creating dataframe dictionary arbitrary length...   \n",
       "140853  another way print multiple list elements list ...   \n",
       "140854  multiply every matrix numpy array transpose ev...   \n",
       "140855  pythonic way handling python code handling nan...   \n",
       "140856                django find pillow though installed   \n",
       "140857  setting textinput focus equal true makes texti...   \n",
       "140858                 display dictionary print statement   \n",
       "140859             pygametransformscale work game surface   \n",
       "140860                       return multi line dictionary   \n",
       "140861  attributeerror file object attribute _ committ...   \n",
       "140862    plotting scatterplots dataframe grid matplotlib   \n",
       "140863  fix error valueerror follower follower object ...   \n",
       "140864       displaying large image next 3 smaller images   \n",
       "140865                 get specific word fron strcontains   \n",
       "140866  installation tensorflow aarch64 error could fi...   \n",
       "140867      need filter data rows data past certain month   \n",
       "140868             find words containing specific letters   \n",
       "140869         focus lost event dispatcher kivy textinput   \n",
       "140870                                shift maximum value   \n",
       "140871  checking two dictionaries key val pairs values...   \n",
       "140872                    unnest explode multiple list 20   \n",
       "140873                          limit users integer input   \n",
       "140874                              stop loop index value   \n",
       "140875       grab value dictionary contains multiple keys   \n",
       "140876              python regex pattern always start end   \n",
       "140877                 use pandas filter unique test loop   \n",
       "140878  way visualize azimuthal average 3d cartesian s...   \n",
       "\n",
       "                                           original_title  overall_scores  \n",
       "0       Using 'in' to match an attribute of Python obj...              84  \n",
       "1               Specifying a mySQL ENUM in a Django model             196  \n",
       "2          Unicode vs UTF-8 confusion in Python / Django?              53  \n",
       "3              Cleanest & Fastest server setup for Django              60  \n",
       "4       Always including the user in the django templa...             102  \n",
       "5                          How to generate urls in django              47  \n",
       "6       Project design / FS layout for large django pr...              37  \n",
       "7       How would you make a comma-separated string fr...             903  \n",
       "8                      What's the best Django search app?             174  \n",
       "9                           Ignore case in Python strings              90  \n",
       "10      When to create a new app (with startapp) in Dj...              88  \n",
       "11              Caching compiled regex objects in Python?              14  \n",
       "12      Serving dynamically generated ZIP archives in ...             118  \n",
       "13      Why are Python's 'private' methods not actuall...            1201  \n",
       "14          How to create a triple-join table with Django              11  \n",
       "15            How to use form values from an unbound form               6  \n",
       "16                   How do I split a string into a list?             142  \n",
       "17      In Python, what is the fastest algorithm for r...             119  \n",
       "18      Django + FCGID on Fedora Core 9 -- what am I m...               1  \n",
       "19      Stripping non printable characters from a stri...             163  \n",
       "20       How do you send a HEAD HTTP request in Python 2?             328  \n",
       "21      Change the width of form elements created with...             126  \n",
       "22      List all words in a dictionary that start with...              41  \n",
       "23                              Python Regex vs PHP Regex              18  \n",
       "24          How can I join a list into a string (caveat)?              10  \n",
       "25      How do you translate this regular-expression i...              69  \n",
       "26                   Adding New Element to Text Substring               4  \n",
       "27                 Editing XML as a dictionary in python?              43  \n",
       "28      What is the best way to store set data in Python?               4  \n",
       "29      Is there an easy way to populate SlugField fro...              72  \n",
       "...                                                   ...             ...  \n",
       "140849  Expand the possibility to put offline a post a...               1  \n",
       "140850  Slicing with :-1 and None - What does each of ...               1  \n",
       "140851          Django: Get first Choice Value from tuple               3  \n",
       "140852  Creating a dataframe from dictionary with arbi...               1  \n",
       "140853  Is there another way to print multiple list el...               0  \n",
       "140854  Multiply every matrix of a numpy array with th...               2  \n",
       "140855  Pythonic way of handling a python code for han...               1  \n",
       "140856       Django does not find Pillow though installed               1  \n",
       "140857  Why is it that setting TextInput focus equal t...               1  \n",
       "140858              Display dictionary in print statement               0  \n",
       "140859  pygame.transform.scale does not work on the \"g...               0  \n",
       "140860                     Return a multi-line dictionary               3  \n",
       "140861  AttributeError: 'file' object has no attribute...               0  \n",
       "140862  Plotting scatterplots from a dataframe on a gr...              -1  \n",
       "140863  How to fix error \"ValueError: <Follower: Follo...               0  \n",
       "140864  Displaying a large image next to 3 smaller images               1  \n",
       "140865     How to get the specific word fron str.contains               2  \n",
       "140866  Installation of Tensorflow on aarch64: Error: ...               0  \n",
       "140867  Need to filter data rows where the data is pas...               4  \n",
       "140868  How to Find Words Not Containing Specific Lett...               3  \n",
       "140869  Is there a focus lost event dispatcher for a k...               0  \n",
       "140870                            Shift the maximum value               0  \n",
       "140871  Checking if two dictionaries of key-val pairs ...              -1  \n",
       "140872                 unnest (explode) multiple list 2.0               0  \n",
       "140873                How to limit a users integer input?               3  \n",
       "140874                  Stop a For Loop at an Index Value               5  \n",
       "140875  How to grab a value from a dictionary which co...               1  \n",
       "140876  In Python, should a regex pattern always start...               4  \n",
       "140877  How can I use a pandas filter and unique for t...               0  \n",
       "140878  Is there a way to visualize an azimuthal avera...               0  \n",
       "\n",
       "[140879 rows x 10 columns]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('Preprocessed_data.csv')\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import saved WordEmbeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import saved Wordvec Embeddings\n",
    "import gensim\n",
    "w2v_model = gensim.models.word2vec.Word2Vec.load('models/SO_word2vec_embeddings.bin')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate Sentence Embeddings\n",
    "In order to calculate the embeddings for an entire sentence, I defined the following function which averages the the embeddings for each valid token "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def question_to_vec(question, embeddings, dim=300):\n",
    "    question_embedding = np.zeros(dim)\n",
    "    valid_words = 0\n",
    "    for word in question.split(' '):\n",
    "        if word in embeddings:\n",
    "            valid_words += 1\n",
    "            question_embedding += embeddings[word]\n",
    "    if valid_words > 0:\n",
    "        return question_embedding/valid_words\n",
    "    else:\n",
    "        return question_embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohit\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\ipykernel_launcher.py:5: DeprecationWarning: Call to deprecated `__contains__` (Method will be removed in 4.0.0, use self.wv.__contains__() instead).\n",
      "  \"\"\"\n",
      "C:\\Users\\Rohit\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\ipykernel_launcher.py:7: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  import sys\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "all_title_embeddings = []\n",
    "for title in data.processed_title:\n",
    "    all_title_embeddings.append(question_to_vec(title, w2v_model))\n",
    "all_title_embeddings = np.array(all_title_embeddings)\n",
    "\n",
    "embeddings = pd.DataFrame(data = all_title_embeddings)\n",
    "embeddings.to_csv('models/title_embeddings.csv', index=False)\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since the number of titles have have is fixed, I saved the sentence embeddings for all titles in a .csv file to save computation time on future runs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_title_embeddings = pd.read_csv('models/title_embeddings.csv').values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import the saved model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras.backend as K\n",
    "\n",
    "# Custom loss function to handle multilabel classification task\n",
    "def multitask_loss(y_true, y_pred):\n",
    "    # Avoid divide by 0\n",
    "    y_pred = K.clip(y_pred, K.epsilon(), 1 - K.epsilon())\n",
    "    # Multi-task loss\n",
    "    return K.mean(K.sum(- y_true * K.log(y_pred) - (1 - y_true) * K.log(1 - y_pred), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "import keras.losses\n",
    "\n",
    "keras.losses.multitask_loss = multitask_loss\n",
    "model = load_model('models/Tag_predictor.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_tags(text, include_neutral=True):\n",
    "    # Tokenize text\n",
    "    x_test = pad_sequences(tokenizer.texts_to_sequences([text]), maxlen=MAX_SEQUENCE_LENGTH)\n",
    "    # Predict\n",
    "    prediction = model.predict([x_test])[0]\n",
    "    for i,value in enumerate(prediction):\n",
    "        if value > 0.5:\n",
    "            prediction[i] = 1\n",
    "        else:\n",
    "            prediction[i] = 0\n",
    "    tags = tag_encoder.inverse_transform(np.array([prediction]))\n",
    "    return tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import nltk\n",
    "import inflect\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "def tokenize_text(text):\n",
    "    \"Apply tokenization using spacy to docstrings.\"\n",
    "    tokens = EN.tokenizer(text)\n",
    "    return [token.text.lower() for token in tokens if not token.is_space]\n",
    "\n",
    "def to_lowercase(words):\n",
    "    \"\"\"Convert all characters to lowercase from list of tokenized words\"\"\"\n",
    "    new_words = []\n",
    "    for word in words:\n",
    "        new_word = word.lower()\n",
    "        new_words.append(new_word)\n",
    "    return new_words\n",
    "\n",
    "def remove_punctuation(words):\n",
    "    \"\"\"Remove punctuation from list of tokenized words\"\"\"\n",
    "    new_words = []\n",
    "    for word in words:\n",
    "        new_word = re.sub(r'[^\\w\\s]', '', word)\n",
    "        if new_word != '':\n",
    "            new_words.append(new_word)\n",
    "    return new_words\n",
    "\n",
    "def remove_stopwords(words):\n",
    "    \"\"\"Remove stop words from list of tokenized words\"\"\"\n",
    "    new_words = []\n",
    "    for word in words:\n",
    "        if word not in stopwords.words('english'):\n",
    "            new_words.append(word)\n",
    "    return new_words\n",
    "\n",
    "def normalize(words):\n",
    "    words = to_lowercase(words)\n",
    "    words = remove_punctuation(words)\n",
    "    words = remove_stopwords(words)\n",
    "    return words\n",
    "\n",
    "def tokenize_code(text):\n",
    "    \"A very basic procedure for tokenizing code strings.\"\n",
    "    return RegexpTokenizer(r'\\w+').tokenize(text)\n",
    "\n",
    "def preprocess_text(text):\n",
    "    return ' '.join(normalize(tokenize_text(text)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import the saved Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 300\n",
    "import pickle\n",
    "with open('models/tokenizer.pickle', 'rb') as handle:\n",
    "    tokenizer = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting the most similar results\n",
    "So the way we actually calculate the the most similar results, is by comparing how far each result is from the query in terms of distance. This can only be done if both the query and the results are in a shared vector space. **Fortunately, that is exactly what our word embeddings are for**. They create each sentence as a vector in the embedding space, which makes it easy for us to distinguish them. \n",
    "\n",
    "After we have those vectors, we can assign a **Similarity Measure** as a metric which measures the closeness of two vectors. Common examples are Cosine distance, Euclidean distance and more.\n",
    "\n",
    "**However, for this specific task, I decided to assign a custom similarity measure**. It is defined as follows:\n",
    "\n",
    "![Similarity Measure](jupyter_imgs/similaritymeasure.png)\n",
    "\n",
    "- It considers the cosine distance as a base measure\n",
    "- It takes into account the popularity of the post based on the votes it has received by users at StackOverflow\n",
    "- It takes into account the overall sentiment of the responses that people have made. A positive sentiment entails that the answers were helpful and thus is a good post "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohit\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\ipykernel_launcher.py:5: DeprecationWarning: Call to deprecated `__contains__` (Method will be removed in 4.0.0, use self.wv.__contains__() instead).\n",
      "  \"\"\"\n",
      "C:\\Users\\Rohit\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\ipykernel_launcher.py:7: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  import sys\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<h3>Results:</h3><a target=\"_blank\" href=https://stackoverflow.com/questions/1720421><h2>How do I concatenate two lists in Python?</h2></a><h3> Similarity Score: 0.9988480985354673</h3><h3> Stackover Votes: 4279</h3><p style=\"font-family:verdana; font-size:110%;\">  concatenate two <b>lists</b> python concatenate two <b>lists</b></p><hr><a target=\"_blank\" href=https://stackoverflow.com/questions/26702846><h2>How to Merge Lists of Lists</h2></a><h3> Similarity Score: 0.9472702607038342</h3><h3> Stackover Votes: 8</h3><p style=\"font-family:verdana; font-size:110%;\">  merge <b>lists</b> <b>lists</b> working python coding project <b>st</b></p><hr><a target=\"_blank\" href=https://stackoverflow.com/questions/48170089><h2>Merge 2 lists of lists</h2></a><h3> Similarity Score: 0.9442272692300363</h3><h3> Stackover Votes: 5</h3><p style=\"font-family:verdana; font-size:110%;\">  merge 2 <b>lists</b> <b>lists</b> im looking merge 2 <b>lists</b> <b>lists</b></p><hr><a target=\"_blank\" href=https://stackoverflow.com/questions/24830804><h2>Convert 2 lists into list of lists</h2></a><h3> Similarity Score: 0.9320485322581505</h3><h3> Stackover Votes: 5</h3><p style=\"font-family:verdana; font-size:110%;\">  convert 2 <b>lists</b> <b>list</b> <b>lists</b> 2 <b>lists</b></p><hr><a target=\"_blank\" href=https://stackoverflow.com/questions/6824212><h2>Python: Merging lists with lists</h2></a><h3> Similarity Score: 0.9311796475042384</h3><h3> Stackover Votes: 2</h3><p style=\"font-family:verdana; font-size:110%;\">  python merging <b>lists</b> <b>lists</b> <b>list</b></p><hr>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import HTML\n",
    "import logging\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "search_string = \"Combine lists of lists\" \n",
    "search_string = ' '.join(normalize(tokenize_text(search_string)))\n",
    "results_returned = \"5\" \n",
    "search_vect = np.array([question_to_vec(search_string, w2v_model)])    # Vectorize the user query\n",
    "\n",
    "# Calculate Cosine similarites for the query and all titles\n",
    "cosine_similarities = pd.Series(cosine_similarity(search_vect, all_title_embeddings)[0])\n",
    "\n",
    "# Custom Similarity Measure\n",
    "cosine_similarities = cosine_similarities*(1 + 0.4*data.overall_scores_norm + 0.1*(data.sentiment_polarity))\n",
    "\n",
    "output =\"\"\n",
    "for i,j in cosine_similarities.nlargest(int(results_returned)).iteritems():\n",
    "    output += '<a target=\"_blank\" href='+ str(data.question_url[i])+'><h2>' + data.original_title[i] + '</h2></a>'\n",
    "    output += '<h3> Similarity Score: ' + str(j) + '</h3>'\n",
    "    output += '<h3> Stackover Votes: ' + str(data.overall_scores[i]) + '</h3>'\n",
    "    output +='<p style=\"font-family:verdana; font-size:110%;\"> '\n",
    "    for i in data.question_content[i][:50].split():\n",
    "        if i.lower() in search_string:\n",
    "            output += \" <b>\"+str(i)+\"</b>\"\n",
    "        else:\n",
    "            output += \" \"+str(i)\n",
    "    output += \"</p><hr>\"\n",
    "    \n",
    "output = '<h3>Results:</h3>'+output\n",
    "display(HTML(output))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}