index.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-03T10:41:50.883723Z",
     "start_time": "2019-10-03T10:41:50.202209Z"
    },
    "hide_input": false,
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container {width:100% !important;}</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2>Loading results of previous run on <u>             Mon Oct  7 20:16:48 2019</u> from ./persistent_storage/20191007.pkl"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 12954 entries, 0 to 12953\n",
      "Data columns (total 9 columns):\n",
      "WkD_disease              12954 non-null object\n",
      "WkD_diseaseLabel         12954 non-null object\n",
      "WP_en_article            4715 non-null object\n",
      "class                    4525 non-null object\n",
      "importance               4515 non-null object\n",
      "taskForces               2600 non-null object\n",
      "cochrane_reviews_html    4715 non-null object\n",
      "cochrane_reviews         4715 non-null object\n",
      "talkPage_categories      4668 non-null object\n",
      "dtypes: object(9)\n",
      "memory usage: 910.9+ KB\n"
     ]
    }
   ],
   "source": [
    "#------------------------------------------------------------------------------------------------------\n",
    "# imports, logger initiation, loading persistant data\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "from IPython.core.display import display, HTML\n",
    "from IPython.display import clear_output, Markdown\n",
    "from IPython.display import FileLink, FileLinks\n",
    "\n",
    "import logging\n",
    "import dill\n",
    "# import qgrid\n",
    "# from wikidata.client import Client\n",
    "\n",
    "import pandas as pd\n",
    "from pandas.compat import StringIO\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import json\n",
    "from SPARQLWrapper import SPARQLWrapper, JSON\n",
    "import ipywidgets as widgets\n",
    "\n",
    "from urllib.parse import unquote\n",
    "from urllib.parse import quote\n",
    "\n",
    "import wikipedia\n",
    "from tqdm import tnrange, tqdm_notebook\n",
    "from tqdm import tqdm\n",
    "\n",
    "import ipywidgets as widgetsM\n",
    "from ipywidgets import HBox, VBox\n",
    "from ipywidgets import Button, Layout\n",
    "\n",
    "from operator import itemgetter\n",
    "from collections import OrderedDict\n",
    "\n",
    "import time\n",
    "from datetime import datetime\n",
    "import html2text\n",
    "import re\n",
    "import sys\n",
    "import os\n",
    "import io\n",
    "\n",
    "from Bio import Entrez\n",
    "from urllib.error import HTTPError\n",
    "\n",
    "# from fuzzywuzzy import fuzz\n",
    "# from fuzzywuzzy import process\n",
    "\n",
    "from  itertools import chain\n",
    "from collections import Counter\n",
    "\n",
    "\n",
    "display(HTML(\"<style>.container {width:100% !important;}</style>\"))\n",
    "\n",
    "logger = logging.getLogger(__name__)\n",
    "logger.setLevel(logging.DEBUG)\n",
    "\n",
    "# create file handler which logs even debug messages\n",
    "fh1 = logging.FileHandler('./logs/log.html',mode='w')\n",
    "fh1.setLevel(logging.DEBUG)\n",
    "\n",
    "# create console handler with a higher log level\n",
    "console_handler = logging.StreamHandler()\n",
    "console_handler.setLevel(logging.ERROR) #------------------change your log setting here DEBUG INFO WARNING ERROR CRITICAL\n",
    "\n",
    "formatter = logging.Formatter('<p> %(asctime)s - %(name)s - %(levelname)s - %(message)s <p>' , datefmt='%d-%b-%y %H:%M:%S')\n",
    "fh1.setFormatter(formatter)\n",
    "console_handler.setFormatter(formatter)\n",
    "\n",
    "# add the handlers to the logger\n",
    "logger.addHandler(fh1)\n",
    "logger.addHandler(console_handler)\n",
    "\n",
    "listOfFiles = os.listdir('./persistent_storage/')\n",
    "pkl_file = (sorted(listOfFiles, reverse=True)[0])\n",
    "\n",
    "display(HTML(f\"<h2>Loading results of previous run on <u>\\\n",
    "             {time.ctime(os.path.getmtime(f'./persistent_storage/{pkl_file}'))}</u> from ./persistent_storage/{pkl_file}\"))\n",
    "logger.info(f\"Loading df from ./persistent_storage/{pkl_file}\")\n",
    "#with open('./persistent_storage/df.dill', 'rb') as in_strm:\n",
    "#     df = dill.load(in_strm)\n",
    "#dill.load_session('./persistent_storage/dill_session')   \n",
    "df=pd.read_pickle(f\"./persistent_storage/{pkl_file}\")\n",
    "df.info()\n",
    "\n",
    "# def on_button_load_clicked(b):\n",
    "#     display(HTML(f\"<h2>Loading df from ./persistent_storage/...\"))\n",
    "#     logger.info(f\"Loading df from ./persistent_storage...\")\n",
    "#     #with open('./persistent_storage/df.dill', 'rb') as in_strm:\n",
    "#     #     df = dill.load(in_strm)\n",
    "#     #dill.load_session('./persistent_storage/dill_session')   \n",
    "#     df=pd.read_pickle(\"./persistent_storage/df.pkl\")\n",
    "#     df.info()\n",
    "#     text_area.value=df.info()\n",
    "\n",
    "    \n",
    "# button_load = widgets.Button(description=\"Load data from persistent storage\",layout=Layout(width='20%', height='80px'))\n",
    "# button_load.style.button_color = 'lightgreen'\n",
    "# button_load.on_click(on_button_load_clicked)\n",
    "# display(button_load)\n",
    "\n",
    "# text_area= widgets.Textarea(value=df.info(),placeholder='',description='Loaded data summary:',disabled=False)\n",
    "# display(text_area)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-03T10:42:00.073241Z",
     "start_time": "2019-10-03T10:41:50.899299Z"
    },
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "# Statistical Summaries:"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Total number of diseases identified in Wikidata: 12954"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Total number of diseases identified in Wikidata which have a corresponding article in Wikipedia:                  4715"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "-----------------------------------------------------"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Wikipedia articles class:"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_de641974_e937_11e9_907b_75f58ebadf75\" ><thead>    <tr>        <th class=\"col_heading level0 col0\" >Article quality class</th>        <th class=\"col_heading level0 col1\" >Total count</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                                <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row0_col0\" class=\"data row0 col0\" >Start</td>\n",
       "                        <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row0_col1\" class=\"data row0 col1\" >1829</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row1_col0\" class=\"data row1 col0\" >Stub</td>\n",
       "                        <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row1_col1\" class=\"data row1 col1\" >1035</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row2_col0\" class=\"data row2 col0\" >C</td>\n",
       "                        <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row2_col1\" class=\"data row2 col1\" >991</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row3_col0\" class=\"data row3 col0\" >B</td>\n",
       "                        <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row3_col1\" class=\"data row3 col1\" >541</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row4_col0\" class=\"data row4 col0\" >nan</td>\n",
       "                        <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row4_col1\" class=\"data row4 col1\" >190</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row5_col0\" class=\"data row5 col0\" >GA</td>\n",
       "                        <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row5_col1\" class=\"data row5 col1\" >79</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row6_col0\" class=\"data row6 col0\" >FA</td>\n",
       "                        <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row6_col1\" class=\"data row6 col1\" >23</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row7_col0\" class=\"data row7 col0\" >List</td>\n",
       "                        <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row7_col1\" class=\"data row7 col1\" >16</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row8_col0\" class=\"data row8 col0\" >Disambig</td>\n",
       "                        <td id=\"T_de641974_e937_11e9_907b_75f58ebadf75row8_col1\" class=\"data row8 col1\" >11</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f46690b7ac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Wikipedia articles importance:"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_de680cb4_e937_11e9_907b_75f58ebadf75\" ><thead>    <tr>        <th class=\"col_heading level0 col0\" >Article quality category</th>        <th class=\"col_heading level0 col1\" >Total count</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                                <td id=\"T_de680cb4_e937_11e9_907b_75f58ebadf75row0_col0\" class=\"data row0 col0\" >3-Mid</td>\n",
       "                        <td id=\"T_de680cb4_e937_11e9_907b_75f58ebadf75row0_col1\" class=\"data row0 col1\" >2151</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de680cb4_e937_11e9_907b_75f58ebadf75row1_col0\" class=\"data row1 col0\" >4-Low</td>\n",
       "                        <td id=\"T_de680cb4_e937_11e9_907b_75f58ebadf75row1_col1\" class=\"data row1 col1\" >2007</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de680cb4_e937_11e9_907b_75f58ebadf75row2_col0\" class=\"data row2 col0\" >2-High</td>\n",
       "                        <td id=\"T_de680cb4_e937_11e9_907b_75f58ebadf75row2_col1\" class=\"data row2 col1\" >292</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de680cb4_e937_11e9_907b_75f58ebadf75row3_col0\" class=\"data row3 col0\" >nan</td>\n",
       "                        <td id=\"T_de680cb4_e937_11e9_907b_75f58ebadf75row3_col1\" class=\"data row3 col1\" >200</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de680cb4_e937_11e9_907b_75f58ebadf75row4_col0\" class=\"data row4 col0\" >1-Top</td>\n",
       "                        <td id=\"T_de680cb4_e937_11e9_907b_75f58ebadf75row4_col1\" class=\"data row4 col1\" >65</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f46690b7b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Wikipedia articles taskForces:"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_de69db52_e937_11e9_907b_75f58ebadf75\" ><thead>    <tr>        <th class=\"col_heading level0 col0\" >taskForce</th>        <th class=\"col_heading level0 col1\" >Total count</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row0_col0\" class=\"data row0 col0\" >NA</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row0_col1\" class=\"data row0 col1\" >2115</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row1_col0\" class=\"data row1 col0\" >Ophthalmology</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row1_col1\" class=\"data row1 col1\" >237</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row2_col0\" class=\"data row2 col0\" >Dermatology</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row2_col1\" class=\"data row2 col1\" >951</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row3_col0\" class=\"data row3 col0\" >Toxicology</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row3_col1\" class=\"data row3 col1\" >11</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row4_col0\" class=\"data row4 col0\" >Cardiology</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row4_col1\" class=\"data row4 col1\" >186</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row5_col0\" class=\"data row5 col0\" >Neurology</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row5_col1\" class=\"data row5 col1\" >398</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row6_col0\" class=\"data row6 col0\" >Pathology</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row6_col1\" class=\"data row6 col1\" >247</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row7_col0\" class=\"data row7 col0\" >Medical genetics</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row7_col1\" class=\"data row7 col1\" >462</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row8_col0\" class=\"data row8 col0\" >Pulmonology</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row8_col1\" class=\"data row8 col1\" >65</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row9_col0\" class=\"data row9 col0\" >Reproductive medicine</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row9_col1\" class=\"data row9 col1\" >72</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row10_col0\" class=\"data row10 col0\" >Hematology-oncology</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row10_col1\" class=\"data row10 col1\" >251</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row11_col0\" class=\"data row11 col0\" >Psychiatry</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row11_col1\" class=\"data row11 col1\" >74</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row12_col0\" class=\"data row12 col0\" >Gastroenterology</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row12_col1\" class=\"data row12 col1\" >52</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row13_col0\" class=\"data row13 col0\" >Nephrology</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row13_col1\" class=\"data row13 col1\" >68</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row14_col0\" class=\"data row14 col0\" >Livestock</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row14_col1\" class=\"data row14 col1\" >2</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row15_col0\" class=\"data row15 col0\" >Radiology</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row15_col1\" class=\"data row15 col1\" >4</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row16_col0\" class=\"data row16 col0\" >Applied Linguistics</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row16_col1\" class=\"data row16 col1\" >1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row17_col0\" class=\"data row17 col0\" >Emergency medicine and EMS</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row17_col1\" class=\"data row17 col1\" >6</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row18_col0\" class=\"data row18 col0\" >Ethics</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row18_col1\" class=\"data row18 col1\" >1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row19_col0\" class=\"data row19 col0\" >Sustainability</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row19_col1\" class=\"data row19 col1\" >1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row20_col0\" class=\"data row20 col0\" >Military logistics and medicine</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row20_col1\" class=\"data row20 col1\" >1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row21_col0\" class=\"data row21 col0\" >World War I</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row21_col1\" class=\"data row21 col1\" >1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row22_col0\" class=\"data row22 col0\" >Balkan military history</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row22_col1\" class=\"data row22 col1\" >1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row23_col0\" class=\"data row23 col0\" >European military history</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row23_col1\" class=\"data row23 col1\" >1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row24_col0\" class=\"data row24 col0\" >German military history</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row24_col1\" class=\"data row24 col1\" >1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row25_col0\" class=\"data row25 col0\" >World War II</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row25_col1\" class=\"data row25 col1\" >1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row26_col0\" class=\"data row26 col0\" >Theoretical Linguistics</td>\n",
       "                        <td id=\"T_de69db52_e937_11e9_907b_75f58ebadf75row26_col1\" class=\"data row26 col1\" >1</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f4668477710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "-----------------------------------------------------"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Number of disease related Wikipedia articles which cite one or more Cochrane reviews:                  1219"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Number of disease related Wikipedia articles which do not cite any  Cochrane reviews:                  3496"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "-----------------------------------------------------"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### PMIDs statistical summary 1 (LATEST VERSIONS of Cochrane reviews found and/or cited):"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_df464452_e937_11e9_907b_75f58ebadf75\" class=\"results\"><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >PMIDs of Cochrane reviews (old versions only)</th>        <th class=\"col_heading level0 col1\" >Title</th>        <th class=\"col_heading level0 col2\" >Times appeared in search results</th>        <th class=\"col_heading level0 col3\" >Times cited in Wikipedia</th>        <th class=\"col_heading level0 col4\" >Version (1=latest, 0=old)</th>        <th class=\"col_heading level0 col5\" >Times cited in a Top Importance WP articles</th>        <th class=\"col_heading level0 col6\" >Times cited in a High Importance WP articles</th>        <th class=\"col_heading level0 col7\" >Times cited in a Mid Importance WP articles</th>        <th class=\"col_heading level0 col8\" >Times cited in a Low Importance WP articles</th>        <th class=\"col_heading level0 col9\" >Times cited in an Unknown Importance WP articles</th>        <th class=\"col_heading level0 col10\" >Times cited in a FA quality WP articles</th>        <th class=\"col_heading level0 col11\" >Times cited in a A quality WP articles</th>        <th class=\"col_heading level0 col12\" >Times cited in a GA quality WP articles</th>        <th class=\"col_heading level0 col13\" >Times cited in a B quality WP articles</th>        <th class=\"col_heading level0 col14\" >Times cited in a C quality WP articles</th>        <th class=\"col_heading level0 col15\" >Times cited in a Start quality WP articles</th>        <th class=\"col_heading level0 col16\" >Times cited in a Stub quality WP articles</th>        <th class=\"col_heading level0 col17\" >Times cited in a Other quality WP articles</th>        <th class=\"col_heading level0 col18\" >Times cited in an Unknown quality WP articles</th>    </tr>    <tr>        <th class=\"index_name level0\" >index</th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_df464452_e937_11e9_907b_75f58ebadf75level0_row0\" class=\"row_heading level0 row0\" >Total</th>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col0\" class=\"data row0 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/N/A\">N/A</a></td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col1\" class=\"data row0 col1\" >N/A</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col2\" class=\"data row0 col2\" >15382</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col3\" class=\"data row0 col3\" >1143</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col4\" class=\"data row0 col4\" >6894</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col5\" class=\"data row0 col5\" >425</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col6\" class=\"data row0 col6\" >391</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col7\" class=\"data row0 col7\" >286</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col8\" class=\"data row0 col8\" >30</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col9\" class=\"data row0 col9\" >11</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col10\" class=\"data row0 col10\" >86</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col11\" class=\"data row0 col11\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col12\" class=\"data row0 col12\" >288</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col13\" class=\"data row0 col13\" >521</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col14\" class=\"data row0 col14\" >169</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col15\" class=\"data row0 col15\" >62</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col16\" class=\"data row0 col16\" >5</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col17\" class=\"data row0 col17\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row0_col18\" class=\"data row0 col18\" >11</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_df464452_e937_11e9_907b_75f58ebadf75level0_row1\" class=\"row_heading level0 row1\" >1545</th>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col0\" class=\"data row1 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/18254088\">18254088</a></td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col1\" class=\"data row1 col1\" >Intravitreal steroids for macular edema in diabetes.</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col2\" class=\"data row1 col2\" >4</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col3\" class=\"data row1 col3\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col4\" class=\"data row1 col4\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col5\" class=\"data row1 col5\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col6\" class=\"data row1 col6\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col7\" class=\"data row1 col7\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col8\" class=\"data row1 col8\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col9\" class=\"data row1 col9\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col10\" class=\"data row1 col10\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col11\" class=\"data row1 col11\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col12\" class=\"data row1 col12\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col13\" class=\"data row1 col13\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col14\" class=\"data row1 col14\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col15\" class=\"data row1 col15\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col16\" class=\"data row1 col16\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col17\" class=\"data row1 col17\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row1_col18\" class=\"data row1 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_df464452_e937_11e9_907b_75f58ebadf75level0_row2\" class=\"row_heading level0 row2\" >5718</th>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col0\" class=\"data row2 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/27245310\">27245310</a></td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col1\" class=\"data row2 col1\" >Speech and language therapy for aphasia following stroke.</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col2\" class=\"data row2 col2\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col3\" class=\"data row2 col3\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col4\" class=\"data row2 col4\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col5\" class=\"data row2 col5\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col6\" class=\"data row2 col6\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col7\" class=\"data row2 col7\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col8\" class=\"data row2 col8\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col9\" class=\"data row2 col9\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col10\" class=\"data row2 col10\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col11\" class=\"data row2 col11\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col12\" class=\"data row2 col12\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col13\" class=\"data row2 col13\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col14\" class=\"data row2 col14\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col15\" class=\"data row2 col15\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col16\" class=\"data row2 col16\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col17\" class=\"data row2 col17\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row2_col18\" class=\"data row2 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_df464452_e937_11e9_907b_75f58ebadf75level0_row3\" class=\"row_heading level0 row3\" >4633</th>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col0\" class=\"data row3 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/26241698\">26241698</a></td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col1\" class=\"data row3 col1\" >Post-pyloric versus gastric tube feeding for preventing pneumonia and improving nutritional outcomes in critically ill adults.</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col2\" class=\"data row3 col2\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col3\" class=\"data row3 col3\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col4\" class=\"data row3 col4\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col5\" class=\"data row3 col5\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col6\" class=\"data row3 col6\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col7\" class=\"data row3 col7\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col8\" class=\"data row3 col8\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col9\" class=\"data row3 col9\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col10\" class=\"data row3 col10\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col11\" class=\"data row3 col11\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col12\" class=\"data row3 col12\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col13\" class=\"data row3 col13\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col14\" class=\"data row3 col14\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col15\" class=\"data row3 col15\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col16\" class=\"data row3 col16\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col17\" class=\"data row3 col17\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row3_col18\" class=\"data row3 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_df464452_e937_11e9_907b_75f58ebadf75level0_row4\" class=\"row_heading level0 row4\" >5015</th>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col0\" class=\"data row4 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/29664187\">29664187</a></td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col1\" class=\"data row4 col1\" >Exercise interventions and patient beliefs for people with hip, knee or hip and knee osteoarthritis: a mixed methods review.</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col2\" class=\"data row4 col2\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col3\" class=\"data row4 col3\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col4\" class=\"data row4 col4\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col5\" class=\"data row4 col5\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col6\" class=\"data row4 col6\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col7\" class=\"data row4 col7\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col8\" class=\"data row4 col8\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col9\" class=\"data row4 col9\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col10\" class=\"data row4 col10\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col11\" class=\"data row4 col11\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col12\" class=\"data row4 col12\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col13\" class=\"data row4 col13\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col14\" class=\"data row4 col14\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col15\" class=\"data row4 col15\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col16\" class=\"data row4 col16\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col17\" class=\"data row4 col17\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row4_col18\" class=\"data row4 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_df464452_e937_11e9_907b_75f58ebadf75level0_row5\" class=\"row_heading level0 row5\" >5022</th>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col0\" class=\"data row5 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/27103611\">27103611</a></td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col1\" class=\"data row5 col1\" >Topical NSAIDs for chronic musculoskeletal pain in adults.</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col2\" class=\"data row5 col2\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col3\" class=\"data row5 col3\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col4\" class=\"data row5 col4\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col5\" class=\"data row5 col5\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col6\" class=\"data row5 col6\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col7\" class=\"data row5 col7\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col8\" class=\"data row5 col8\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col9\" class=\"data row5 col9\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col10\" class=\"data row5 col10\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col11\" class=\"data row5 col11\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col12\" class=\"data row5 col12\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col13\" class=\"data row5 col13\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col14\" class=\"data row5 col14\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col15\" class=\"data row5 col15\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col16\" class=\"data row5 col16\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col17\" class=\"data row5 col17\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row5_col18\" class=\"data row5 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_df464452_e937_11e9_907b_75f58ebadf75level0_row6\" class=\"row_heading level0 row6\" >4713</th>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col0\" class=\"data row6 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/26824399\">26824399</a></td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col1\" class=\"data row6 col1\" >Surgical versus non-surgical treatment for lumbar spinal stenosis.</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col2\" class=\"data row6 col2\" >3</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col3\" class=\"data row6 col3\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col4\" class=\"data row6 col4\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col5\" class=\"data row6 col5\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col6\" class=\"data row6 col6\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col7\" class=\"data row6 col7\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col8\" class=\"data row6 col8\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col9\" class=\"data row6 col9\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col10\" class=\"data row6 col10\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col11\" class=\"data row6 col11\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col12\" class=\"data row6 col12\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col13\" class=\"data row6 col13\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col14\" class=\"data row6 col14\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col15\" class=\"data row6 col15\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col16\" class=\"data row6 col16\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col17\" class=\"data row6 col17\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row6_col18\" class=\"data row6 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_df464452_e937_11e9_907b_75f58ebadf75level0_row7\" class=\"row_heading level0 row7\" >1148</th>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col0\" class=\"data row7 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/16856036\">16856036</a></td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col1\" class=\"data row7 col1\" >Self-help and guided self-help for eating disorders.</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col2\" class=\"data row7 col2\" >3</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col3\" class=\"data row7 col3\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col4\" class=\"data row7 col4\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col5\" class=\"data row7 col5\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col6\" class=\"data row7 col6\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col7\" class=\"data row7 col7\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col8\" class=\"data row7 col8\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col9\" class=\"data row7 col9\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col10\" class=\"data row7 col10\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col11\" class=\"data row7 col11\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col12\" class=\"data row7 col12\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col13\" class=\"data row7 col13\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col14\" class=\"data row7 col14\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col15\" class=\"data row7 col15\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col16\" class=\"data row7 col16\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col17\" class=\"data row7 col17\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row7_col18\" class=\"data row7 col18\" >1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_df464452_e937_11e9_907b_75f58ebadf75level0_row8\" class=\"row_heading level0 row8\" >2661</th>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col0\" class=\"data row8 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/20927726\">20927726</a></td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col1\" class=\"data row8 col1\" >Grommets (ventilation tubes) for hearing loss associated with otitis media with effusion in children.</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col2\" class=\"data row8 col2\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col3\" class=\"data row8 col3\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col4\" class=\"data row8 col4\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col5\" class=\"data row8 col5\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col6\" class=\"data row8 col6\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col7\" class=\"data row8 col7\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col8\" class=\"data row8 col8\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col9\" class=\"data row8 col9\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col10\" class=\"data row8 col10\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col11\" class=\"data row8 col11\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col12\" class=\"data row8 col12\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col13\" class=\"data row8 col13\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col14\" class=\"data row8 col14\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col15\" class=\"data row8 col15\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col16\" class=\"data row8 col16\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col17\" class=\"data row8 col17\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row8_col18\" class=\"data row8 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_df464452_e937_11e9_907b_75f58ebadf75level0_row9\" class=\"row_heading level0 row9\" >6254</th>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col0\" class=\"data row9 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/24170669\">24170669</a></td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col1\" class=\"data row9 col1\" >Blood pressure targets for hypertension in people with diabetes mellitus.</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col2\" class=\"data row9 col2\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col3\" class=\"data row9 col3\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col4\" class=\"data row9 col4\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col5\" class=\"data row9 col5\" >2</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col6\" class=\"data row9 col6\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col7\" class=\"data row9 col7\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col8\" class=\"data row9 col8\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col9\" class=\"data row9 col9\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col10\" class=\"data row9 col10\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col11\" class=\"data row9 col11\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col12\" class=\"data row9 col12\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col13\" class=\"data row9 col13\" >1</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col14\" class=\"data row9 col14\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col15\" class=\"data row9 col15\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col16\" class=\"data row9 col16\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col17\" class=\"data row9 col17\" >0</td>\n",
       "                        <td id=\"T_df464452_e937_11e9_907b_75f58ebadf75row9_col18\" class=\"data row9 col18\" >0</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f46690b7ac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2>⋮</h2>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<H3>See the complete table here (sort, filter, and search):</H3>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<b><a href='./results/PMIDs_latestVersions_only.html' target='_blank'>./results/PMIDs_latestVersions_only.html</a></b>"
      ],
      "text/plain": [
       "/home/arash/Desktop/WP2Cochrane/results/PMIDs_latestVersions_only.html"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "-----------------------------------------------------"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### PMIDs statistical summary 2 (OLD VERSIONS of Cochrane reviews found and/or cited):"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75\" class=\"results\"><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >PMIDs of Cochrane reviews (old versions only)</th>        <th class=\"col_heading level0 col1\" >Title</th>        <th class=\"col_heading level0 col2\" >Times appeared in search results</th>        <th class=\"col_heading level0 col3\" >Times cited in Wikipedia</th>        <th class=\"col_heading level0 col4\" >Version (1=latest, 0=old)</th>        <th class=\"col_heading level0 col5\" >Times cited in a Top Importance WP articles</th>        <th class=\"col_heading level0 col6\" >Times cited in a High Importance WP articles</th>        <th class=\"col_heading level0 col7\" >Times cited in a Mid Importance WP articles</th>        <th class=\"col_heading level0 col8\" >Times cited in a Low Importance WP articles</th>        <th class=\"col_heading level0 col9\" >Times cited in an Unknown Importance WP articles</th>        <th class=\"col_heading level0 col10\" >Times cited in a FA quality WP articles</th>        <th class=\"col_heading level0 col11\" >Times cited in a A quality WP articles</th>        <th class=\"col_heading level0 col12\" >Times cited in a GA quality WP articles</th>        <th class=\"col_heading level0 col13\" >Times cited in a B quality WP articles</th>        <th class=\"col_heading level0 col14\" >Times cited in a C quality WP articles</th>        <th class=\"col_heading level0 col15\" >Times cited in a Start quality WP articles</th>        <th class=\"col_heading level0 col16\" >Times cited in a Stub quality WP articles</th>        <th class=\"col_heading level0 col17\" >Times cited in a Other quality WP articles</th>        <th class=\"col_heading level0 col18\" >Times cited in an Unknown quality WP articles</th>    </tr>    <tr>        <th class=\"index_name level0\" >index</th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75level0_row0\" class=\"row_heading level0 row0\" >Total</th>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col0\" class=\"data row0 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/N/A\">N/A</a></td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col1\" class=\"data row0 col1\" >N/A</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col2\" class=\"data row0 col2\" >5103</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col3\" class=\"data row0 col3\" >34</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col4\" class=\"data row0 col4\" >2770</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col5\" class=\"data row0 col5\" >4</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col6\" class=\"data row0 col6\" >10</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col7\" class=\"data row0 col7\" >15</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col8\" class=\"data row0 col8\" >4</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col9\" class=\"data row0 col9\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col10\" class=\"data row0 col10\" >2</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col11\" class=\"data row0 col11\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col12\" class=\"data row0 col12\" >6</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col13\" class=\"data row0 col13\" >11</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col14\" class=\"data row0 col14\" >11</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col15\" class=\"data row0 col15\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col16\" class=\"data row0 col16\" >2</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col17\" class=\"data row0 col17\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row0_col18\" class=\"data row0 col18\" >1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75level0_row1\" class=\"row_heading level0 row1\" >110</th>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col0\" class=\"data row1 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/24142399\">24142399</a></td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col1\" class=\"data row1 col1\" >Pharmacological treatment for pain in Guillain-Barré syndrome.</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col2\" class=\"data row1 col2\" >14</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col3\" class=\"data row1 col3\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col4\" class=\"data row1 col4\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col5\" class=\"data row1 col5\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col6\" class=\"data row1 col6\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col7\" class=\"data row1 col7\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col8\" class=\"data row1 col8\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col9\" class=\"data row1 col9\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col10\" class=\"data row1 col10\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col11\" class=\"data row1 col11\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col12\" class=\"data row1 col12\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col13\" class=\"data row1 col13\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col14\" class=\"data row1 col14\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col15\" class=\"data row1 col15\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col16\" class=\"data row1 col16\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col17\" class=\"data row1 col17\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row1_col18\" class=\"data row1 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75level0_row2\" class=\"row_heading level0 row2\" >480</th>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col0\" class=\"data row2 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/25102015\">25102015</a></td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col1\" class=\"data row2 col1\" >Antioxidant supplementation for lung disease in cystic fibrosis.</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col2\" class=\"data row2 col2\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col3\" class=\"data row2 col3\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col4\" class=\"data row2 col4\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col5\" class=\"data row2 col5\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col6\" class=\"data row2 col6\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col7\" class=\"data row2 col7\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col8\" class=\"data row2 col8\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col9\" class=\"data row2 col9\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col10\" class=\"data row2 col10\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col11\" class=\"data row2 col11\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col12\" class=\"data row2 col12\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col13\" class=\"data row2 col13\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col14\" class=\"data row2 col14\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col15\" class=\"data row2 col15\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col16\" class=\"data row2 col16\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col17\" class=\"data row2 col17\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row2_col18\" class=\"data row2 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75level0_row3\" class=\"row_heading level0 row3\" >1943</th>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col0\" class=\"data row3 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/26174592\">26174592</a></td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col1\" class=\"data row3 col1\" >Assisted reproductive technology: an overview of Cochrane Reviews.</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col2\" class=\"data row3 col2\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col3\" class=\"data row3 col3\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col4\" class=\"data row3 col4\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col5\" class=\"data row3 col5\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col6\" class=\"data row3 col6\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col7\" class=\"data row3 col7\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col8\" class=\"data row3 col8\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col9\" class=\"data row3 col9\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col10\" class=\"data row3 col10\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col11\" class=\"data row3 col11\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col12\" class=\"data row3 col12\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col13\" class=\"data row3 col13\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col14\" class=\"data row3 col14\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col15\" class=\"data row3 col15\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col16\" class=\"data row3 col16\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col17\" class=\"data row3 col17\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row3_col18\" class=\"data row3 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75level0_row4\" class=\"row_heading level0 row4\" >1242</th>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col0\" class=\"data row4 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/27089005\">27089005</a></td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col1\" class=\"data row4 col1\" >Non-absorbable disaccharides versus placebo/no intervention and lactulose versus lactitol for the prevention and treatment of hepatic encephalopathy in people with cirrhosis.</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col2\" class=\"data row4 col2\" >5</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col3\" class=\"data row4 col3\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col4\" class=\"data row4 col4\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col5\" class=\"data row4 col5\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col6\" class=\"data row4 col6\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col7\" class=\"data row4 col7\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col8\" class=\"data row4 col8\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col9\" class=\"data row4 col9\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col10\" class=\"data row4 col10\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col11\" class=\"data row4 col11\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col12\" class=\"data row4 col12\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col13\" class=\"data row4 col13\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col14\" class=\"data row4 col14\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col15\" class=\"data row4 col15\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col16\" class=\"data row4 col16\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col17\" class=\"data row4 col17\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row4_col18\" class=\"data row4 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75level0_row5\" class=\"row_heading level0 row5\" >2554</th>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col0\" class=\"data row5 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/22161393\">22161393</a></td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col1\" class=\"data row5 col1\" >Progestogen for treating threatened miscarriage.</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col2\" class=\"data row5 col2\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col3\" class=\"data row5 col3\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col4\" class=\"data row5 col4\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col5\" class=\"data row5 col5\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col6\" class=\"data row5 col6\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col7\" class=\"data row5 col7\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col8\" class=\"data row5 col8\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col9\" class=\"data row5 col9\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col10\" class=\"data row5 col10\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col11\" class=\"data row5 col11\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col12\" class=\"data row5 col12\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col13\" class=\"data row5 col13\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col14\" class=\"data row5 col14\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col15\" class=\"data row5 col15\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col16\" class=\"data row5 col16\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col17\" class=\"data row5 col17\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row5_col18\" class=\"data row5 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75level0_row6\" class=\"row_heading level0 row6\" >502</th>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col0\" class=\"data row6 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/27552284\">27552284</a></td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col1\" class=\"data row6 col1\" >Inhaled corticosteroids for cystic fibrosis.</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col2\" class=\"data row6 col2\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col3\" class=\"data row6 col3\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col4\" class=\"data row6 col4\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col5\" class=\"data row6 col5\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col6\" class=\"data row6 col6\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col7\" class=\"data row6 col7\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col8\" class=\"data row6 col8\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col9\" class=\"data row6 col9\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col10\" class=\"data row6 col10\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col11\" class=\"data row6 col11\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col12\" class=\"data row6 col12\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col13\" class=\"data row6 col13\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col14\" class=\"data row6 col14\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col15\" class=\"data row6 col15\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col16\" class=\"data row6 col16\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col17\" class=\"data row6 col17\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row6_col18\" class=\"data row6 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75level0_row7\" class=\"row_heading level0 row7\" >1229</th>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col0\" class=\"data row7 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/27884041\">27884041</a></td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col1\" class=\"data row7 col1\" >Follow-up strategies for patients treated for non-metastatic colorectal cancer.</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col2\" class=\"data row7 col2\" >2</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col3\" class=\"data row7 col3\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col4\" class=\"data row7 col4\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col5\" class=\"data row7 col5\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col6\" class=\"data row7 col6\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col7\" class=\"data row7 col7\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col8\" class=\"data row7 col8\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col9\" class=\"data row7 col9\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col10\" class=\"data row7 col10\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col11\" class=\"data row7 col11\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col12\" class=\"data row7 col12\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col13\" class=\"data row7 col13\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col14\" class=\"data row7 col14\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col15\" class=\"data row7 col15\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col16\" class=\"data row7 col16\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col17\" class=\"data row7 col17\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row7_col18\" class=\"data row7 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75level0_row8\" class=\"row_heading level0 row8\" >199</th>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col0\" class=\"data row8 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/26932750\">26932750</a></td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col1\" class=\"data row8 col1\" >Anti-vascular endothelial growth factor (VEGF) drugs for treatment of retinopathy of prematurity.</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col2\" class=\"data row8 col2\" >5</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col3\" class=\"data row8 col3\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col4\" class=\"data row8 col4\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col5\" class=\"data row8 col5\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col6\" class=\"data row8 col6\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col7\" class=\"data row8 col7\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col8\" class=\"data row8 col8\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col9\" class=\"data row8 col9\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col10\" class=\"data row8 col10\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col11\" class=\"data row8 col11\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col12\" class=\"data row8 col12\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col13\" class=\"data row8 col13\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col14\" class=\"data row8 col14\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col15\" class=\"data row8 col15\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col16\" class=\"data row8 col16\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col17\" class=\"data row8 col17\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row8_col18\" class=\"data row8 col18\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75level0_row9\" class=\"row_heading level0 row9\" >530</th>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col0\" class=\"data row9 col0\" ><a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/25093421\">25093421</a></td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col1\" class=\"data row9 col1\" >Pneumococcal vaccines for cystic fibrosis.</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col2\" class=\"data row9 col2\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col3\" class=\"data row9 col3\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col4\" class=\"data row9 col4\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col5\" class=\"data row9 col5\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col6\" class=\"data row9 col6\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col7\" class=\"data row9 col7\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col8\" class=\"data row9 col8\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col9\" class=\"data row9 col9\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col10\" class=\"data row9 col10\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col11\" class=\"data row9 col11\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col12\" class=\"data row9 col12\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col13\" class=\"data row9 col13\" >1</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col14\" class=\"data row9 col14\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col15\" class=\"data row9 col15\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col16\" class=\"data row9 col16\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col17\" class=\"data row9 col17\" >0</td>\n",
       "                        <td id=\"T_e26c4b18_e937_11e9_907b_75f58ebadf75row9_col18\" class=\"data row9 col18\" >0</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f46690b7c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2>⋮</h2>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<H3>See the complete table here (sort, filter, and search):</H3>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<b><a href='./results/PMIDs_oldVersions_only.html' target='_blank'>./results/PMIDs_oldVersions_only.html</a></b>"
      ],
      "text/plain": [
       "/home/arash/Desktop/WP2Cochrane/results/PMIDs_oldVersions_only.html"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "-----------------------------------------------------"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#------------------------------------------------------------------------------------------------------\n",
    "# Statistical summary generator \n",
    "#------------------------------------------------------------------------------------------------------\n",
    "datatables_js_script=\"\"\"\n",
    "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.datatables.net/1.10.19/css/jquery.dataTables.css\">\n",
    "<script src=\"https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js\"></script>\n",
    "<script type=\"text/javascript\" charset=\"utf8\" src=\"https://cdn.datatables.net/1.10.19/js/jquery.dataTables.js\"></script>\n",
    "<script>\n",
    "$(document).ready( function () {\n",
    "    $('table.results').dataTable({\n",
    "    \"lengthMenu\": [[10, 25, 50, -1], [10, 25, 50, \"All\"]], \"bAutoWidth\": false, \n",
    "\n",
    "    });\n",
    "    //$('table.dataframe').dataTable();    \n",
    "} );\n",
    "</script>\n",
    "\"\"\"\n",
    "\n",
    "def make_clickable_PMID(val):\n",
    "    # target _blank to open new window\n",
    "    if not val: \n",
    "        return None\n",
    "    else:\n",
    "        return f'<a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/{val}\">{val}</a>'\n",
    "\n",
    "def imp2int(WP_imp,targetImportance):\n",
    "    val=0\n",
    "    if not WP_imp:\n",
    "        if targetImportance==\"Unknown\":\n",
    "            val=1\n",
    "    else:\n",
    "        if targetImportance==WP_imp:\n",
    "            val=1\n",
    "    return val\n",
    "\n",
    "def class2int(WP_class,targetClass):\n",
    "    val=0\n",
    "    if not WP_class:\n",
    "        if targetClass==\"Unknown\":\n",
    "            val=1\n",
    "    else:\n",
    "        if targetClass==WP_class or (targetClass==\"Other\" and WP_class in ['List','Disambig','FL']):\n",
    "            val=1\n",
    "    return val        \n",
    "\n",
    "display(Markdown(\"# Statistical Summaries:\"))\n",
    "display(Markdown(f\"### Total number of diseases identified in Wikidata: {df.shape[0]}\"))\n",
    "display(Markdown(f\"### Total number of diseases identified in Wikidata which have a corresponding article in Wikipedia:\\\n",
    "                  {df[df['WP_en_article'].notnull()].shape[0]}\"))\n",
    "display(Markdown(\"-----------------------------------------------------\"))                  \n",
    "\n",
    "display(Markdown(\"### Wikipedia articles class:\"))\n",
    "display(df.loc[df['WP_en_article'].notnull()]['class'].value_counts(dropna=False).to_frame().reset_index().rename_axis('index').\\\n",
    "        rename(columns = {'class': 'Total count','index': 'Article quality class'}).\\\n",
    "        sort_values(by = 'Total count', ascending = False).style.hide_index()) \n",
    "\n",
    "display(Markdown(\"### Wikipedia articles importance:\"))\n",
    "display(df.loc[df['WP_en_article'].notnull()]['importance'].value_counts(dropna=False).to_frame().reset_index().rename_axis('index').\\\n",
    "        rename(columns = {'importance': 'Total count','index': 'Article quality category'}).\\\n",
    "        sort_values(by = 'Total count', ascending = False).style.hide_index())\n",
    "\n",
    "\n",
    "display(Markdown('### Wikipedia articles taskForces:'))\n",
    "#display(pd.Series(Counter(chain.from_iterable( df.loc[df['taskForces'].notnull()]['taskForces'] ) )))\n",
    "taskForces_count = {'NA':0}\n",
    "for taskForces in df.loc[df['WP_en_article'].notnull()]['taskForces']:\n",
    "    if taskForces:\n",
    "        for taskForce in taskForces:\n",
    "            if taskForce in taskForces_count:\n",
    "                taskForces_count[taskForce] += 1\n",
    "            else :\n",
    "              taskForces_count[taskForce] = 1\n",
    "    else:\n",
    "        taskForces_count['NA'] += 1\n",
    "display(pd.DataFrame(taskForces_count.items(), columns = ['taskForce', 'Total count']).style.hide_index())\n",
    "display(Markdown(\"-----------------------------------------------------\")) \n",
    "\n",
    "                \n",
    "display(Markdown(f\"### Number of disease related Wikipedia articles which cite one or more Cochrane reviews: \\\n",
    "                 {df[df['cochrane_reviews'].notnull()].shape[0] - df[(df['cochrane_reviews']=={})].shape[0] }\"))\n",
    "display(Markdown(f\"### Number of disease related Wikipedia articles which do not cite any  Cochrane reviews: \\\n",
    "                 { df[(df['cochrane_reviews']=={})].shape[0] }\"))\n",
    "display(Markdown(\"-----------------------------------------------------\")) \n",
    "\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "# PIMD stats  - PMIDs of Cochrane reviews (LATEST versions only)\n",
    "#------------------------------------------------------------------------------------------------------                 \n",
    "uniquePMIDs_count={}\n",
    "for index, row in df.iterrows():                 \n",
    "    WP_class = row['class']\n",
    "    WP_imp = row['importance']\n",
    "    PIMD_paper_dict=row['cochrane_reviews']\n",
    "    if PIMD_paper_dict:\n",
    "        for PMID, paper in PIMD_paper_dict.items():\n",
    "            title = paper[0]\n",
    "            cited = paper[1]\n",
    "            latestVersion = paper[2]\n",
    "            if latestVersion:\n",
    "                if PMID in uniquePMIDs_count:\n",
    "                   uniquePMIDs_count[PMID]=[title, uniquePMIDs_count[PMID][1]+1, uniquePMIDs_count[PMID][2]+int(cited),int(latestVersion)\\\n",
    "                                           , uniquePMIDs_count[PMID][4]+(imp2int(WP_imp,'1-Top')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][5]+(imp2int(WP_imp,'2-High')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][6]+(imp2int(WP_imp,'3-Mid')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][7]+(imp2int(WP_imp,'4-Low')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][8]+(imp2int(WP_imp,'Unknown')*int(cited))                                            \n",
    "                                           , uniquePMIDs_count[PMID][9]+(class2int(WP_class,'FA')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][10]+(class2int(WP_class,'A')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][11]+(class2int(WP_class,'GA')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][12]+(class2int(WP_class,'B')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][13]+(class2int(WP_class,'C')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][14]+(class2int(WP_class,'Start')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][15]+(class2int(WP_class,'Stub')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][16]+(class2int(WP_class,'Other')*int(cited))                                            \n",
    "                                           , uniquePMIDs_count[PMID][17]+(class2int(WP_class,'Unknown')*int(cited))]\n",
    "                else:\n",
    "                   uniquePMIDs_count[PMID]=[title,                            1,                            int(cited),int(latestVersion)\n",
    "                                           , imp2int(WP_imp,'1-Top')*int(cited)\n",
    "                                           , imp2int(WP_imp,'2-High')*int(cited)\n",
    "                                           , imp2int(WP_imp,'3-Mid')*int(cited)\n",
    "                                           , imp2int(WP_imp,'4-Low')*int(cited)\n",
    "                                           , imp2int(WP_imp,'Unknown')*int(cited)\n",
    "                                           , class2int(WP_class,'FA')*int(cited)\n",
    "                                           , class2int(WP_class,'A')*int(cited)\n",
    "                                           , class2int(WP_class,'GA')*int(cited)\n",
    "                                           , class2int(WP_class,'B')*int(cited)\n",
    "                                           , class2int(WP_class,'C')*int(cited)\n",
    "                                           , class2int(WP_class,'Start')*int(cited)\n",
    "                                           , class2int(WP_class,'Stub')*int(cited)\n",
    "                                           , class2int(WP_class,'Other')*int(cited)\n",
    "                                           , class2int(WP_class,'Unknown')*int(cited)]\n",
    "\n",
    "display(Markdown(f'### PMIDs statistical summary 1 (LATEST VERSIONS of Cochrane reviews found and/or cited):'))\n",
    "PMIDs_df= pd.DataFrame.from_dict(uniquePMIDs_count,orient='index').reset_index().rename_axis('index')\\\n",
    "        .rename(columns = {'index': 'PMIDs of Cochrane reviews (old versions only)'\n",
    "                           ,0: 'Title'\n",
    "                           ,1: 'Times appeared in search results'\n",
    "                           ,2: 'Times cited in Wikipedia'\n",
    "                           ,3: 'Version (1=latest, 0=old)'\n",
    "                           ,4: 'Times cited in a Top Importance WP articles'\n",
    "                           ,5: 'Times cited in a High Importance WP articles'\n",
    "                           ,6: 'Times cited in a Mid Importance WP articles'\n",
    "                           ,7: 'Times cited in a Low Importance WP articles'\n",
    "                           ,8: 'Times cited in an Unknown Importance WP articles'\n",
    "                           ,9: 'Times cited in a FA quality WP articles'\n",
    "                           ,10: 'Times cited in a A quality WP articles'\n",
    "                           ,11: 'Times cited in a GA quality WP articles'\n",
    "                           ,12: 'Times cited in a B quality WP articles'\n",
    "                           ,13: 'Times cited in a C quality WP articles'\n",
    "                           ,14: 'Times cited in a Start quality WP articles'\n",
    "                           ,15: 'Times cited in a Stub quality WP articles'\n",
    "                           ,16: 'Times cited in a Other quality WP articles'\n",
    "                           ,17: 'Times cited in an Unknown quality WP articles'\n",
    "                          }).sort_values(by = 'Times cited in Wikipedia', ascending = False)\n",
    "PMIDs_df.loc['Total'] = ['N/A','N/A',PMIDs_df['Times appeared in search results'].sum()\\\n",
    "                         ,PMIDs_df['Times cited in Wikipedia'].sum()\n",
    "                         #,PMIDs_df['Latest version (1=Yes, 0=No)'].sum()\n",
    "                         ,PMIDs_df['Version (1=latest, 0=old)'].count()\n",
    "                         ,PMIDs_df['Times cited in a Top Importance WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a High Importance WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a Mid Importance WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a Low Importance WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in an Unknown Importance WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a FA quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a A quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a GA quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a B quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a C quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a Start quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a Stub quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a Other quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in an Unknown quality WP articles'].sum()\n",
    "                        ]\n",
    "display(PMIDs_df.sort_values(by = 'Times cited in Wikipedia', ascending = False).head(10)\n",
    "        .style.set_table_attributes('class=\"results\"')\\\n",
    "        .format({'PMIDs of Cochrane reviews (old versions only)': make_clickable_PMID}))\n",
    "display(HTML('<h2>⋮</h2>'))\n",
    "f=open(\"./results/PMIDs_latestVersions_only.html\",\"w\")\n",
    "f.write(datatables_js_script + PMIDs_df.sort_values(by = 'Times cited in Wikipedia', ascending = False)\n",
    "        .style.set_table_attributes('class=\"results\"')\\\n",
    "        .format({'PMIDs of Cochrane reviews (latest versions only)': make_clickable_PMID}).render())\n",
    "f.close()\n",
    "display(HTML('<H3>See the complete table here (sort, filter, and search):</H3>')\n",
    "        ,FileLink('./results/PMIDs_latestVersions_only.html', result_html_prefix='<b>', result_html_suffix='</b>'))                 \n",
    "display(Markdown(\"-----------------------------------------------------\"))                \n",
    "\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "# PIMD stats  - PMIDs of Cochrane reviews (old versions only)\n",
    "#------------------------------------------------------------------------------------------------------                 \n",
    "uniquePMIDs_count={}\n",
    "for index, row in df.iterrows():                 \n",
    "    WP_class = row['class']\n",
    "    WP_imp = row['importance']\n",
    "    PIMD_paper_dict=row['cochrane_reviews']\n",
    "    if PIMD_paper_dict:\n",
    "        for PMID, paper in PIMD_paper_dict.items():\n",
    "            title = paper[0]\n",
    "            cited = paper[1]\n",
    "            latestVersion = paper[2]\n",
    "            if not latestVersion:\n",
    "                if PMID in uniquePMIDs_count:\n",
    "                   uniquePMIDs_count[PMID]=[title, uniquePMIDs_count[PMID][1]+1, uniquePMIDs_count[PMID][2]+int(cited),int(latestVersion)\\\n",
    "                                           , uniquePMIDs_count[PMID][4]+(imp2int(WP_imp,'1-Top')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][5]+(imp2int(WP_imp,'2-High')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][6]+(imp2int(WP_imp,'3-Mid')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][7]+(imp2int(WP_imp,'4-Low')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][8]+(imp2int(WP_imp,'Unknown')*int(cited))                                            \n",
    "                                           , uniquePMIDs_count[PMID][9]+(class2int(WP_class,'FA')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][10]+(class2int(WP_class,'A')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][11]+(class2int(WP_class,'GA')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][12]+(class2int(WP_class,'B')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][13]+(class2int(WP_class,'C')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][14]+(class2int(WP_class,'Start')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][15]+(class2int(WP_class,'Stub')*int(cited))\n",
    "                                           , uniquePMIDs_count[PMID][16]+(class2int(WP_class,'Other')*int(cited))                                            \n",
    "                                           , uniquePMIDs_count[PMID][17]+(class2int(WP_class,'Unknown')*int(cited))]\n",
    "                else:\n",
    "                   uniquePMIDs_count[PMID]=[title,                            1,                            int(cited),int(latestVersion)\n",
    "                                           , imp2int(WP_imp,'1-Top')*int(cited)\n",
    "                                           , imp2int(WP_imp,'2-High')*int(cited)\n",
    "                                           , imp2int(WP_imp,'3-Mid')*int(cited)\n",
    "                                           , imp2int(WP_imp,'4-Low')*int(cited)\n",
    "                                           , imp2int(WP_imp,'Unknown')*int(cited)\n",
    "                                           , class2int(WP_class,'FA')*int(cited)\n",
    "                                           , class2int(WP_class,'A')*int(cited)\n",
    "                                           , class2int(WP_class,'GA')*int(cited)\n",
    "                                           , class2int(WP_class,'B')*int(cited)\n",
    "                                           , class2int(WP_class,'C')*int(cited)\n",
    "                                           , class2int(WP_class,'Start')*int(cited)\n",
    "                                           , class2int(WP_class,'Stub')*int(cited)\n",
    "                                           , class2int(WP_class,'Other')*int(cited)\n",
    "                                           , class2int(WP_class,'Unknown')*int(cited)]\n",
    "\n",
    "display(Markdown(f'### PMIDs statistical summary 2 (OLD VERSIONS of Cochrane reviews found and/or cited):'))\n",
    "PMIDs_df= pd.DataFrame.from_dict(uniquePMIDs_count,orient='index').reset_index().rename_axis('index')\\\n",
    "        .rename(columns = {'index': 'PMIDs of Cochrane reviews (old versions only)'\n",
    "                           ,0: 'Title'\n",
    "                           ,1: 'Times appeared in search results'\n",
    "                           ,2: 'Times cited in Wikipedia'\n",
    "                           ,3: 'Version (1=latest, 0=old)'\n",
    "                           ,4: 'Times cited in a Top Importance WP articles'\n",
    "                           ,5: 'Times cited in a High Importance WP articles'\n",
    "                           ,6: 'Times cited in a Mid Importance WP articles'\n",
    "                           ,7: 'Times cited in a Low Importance WP articles'\n",
    "                           ,8: 'Times cited in an Unknown Importance WP articles'\n",
    "                           ,9: 'Times cited in a FA quality WP articles'\n",
    "                           ,10: 'Times cited in a A quality WP articles'\n",
    "                           ,11: 'Times cited in a GA quality WP articles'\n",
    "                           ,12: 'Times cited in a B quality WP articles'\n",
    "                           ,13: 'Times cited in a C quality WP articles'\n",
    "                           ,14: 'Times cited in a Start quality WP articles'\n",
    "                           ,15: 'Times cited in a Stub quality WP articles'\n",
    "                           ,16: 'Times cited in a Other quality WP articles'\n",
    "                           ,17: 'Times cited in an Unknown quality WP articles'\n",
    "                          }).sort_values(by = 'Times cited in Wikipedia', ascending = False)\n",
    "PMIDs_df.loc['Total'] = ['N/A','N/A',PMIDs_df['Times appeared in search results'].sum()\\\n",
    "                         ,PMIDs_df['Times cited in Wikipedia'].sum()\n",
    "                         #,PMIDs_df['Latest version (1=Yes, 0=No)'].sum()\n",
    "                         ,PMIDs_df['Version (1=latest, 0=old)'].count()\n",
    "                         ,PMIDs_df['Times cited in a Top Importance WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a High Importance WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a Mid Importance WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a Low Importance WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in an Unknown Importance WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a FA quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a A quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a GA quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a B quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a C quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a Start quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a Stub quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in a Other quality WP articles'].sum()\n",
    "                         ,PMIDs_df['Times cited in an Unknown quality WP articles'].sum()\n",
    "                        ]\n",
    "display(PMIDs_df.sort_values(by = 'Times cited in Wikipedia', ascending = False).head(10)\n",
    "        .style.set_table_attributes('class=\"results\"')\\\n",
    "        .format({'PMIDs of Cochrane reviews (old versions only)': make_clickable_PMID}))\n",
    "display(HTML('<h2>⋮</h2>'))\n",
    "f=open(\"./results/PMIDs_oldVersions_only.html\",\"w\")\n",
    "f.write(datatables_js_script + PMIDs_df.sort_values(by = 'Times cited in Wikipedia', ascending = False)\n",
    "        .style.set_table_attributes('class=\"results\"')\\\n",
    "        .format({'PMIDs of Cochrane reviews (old versions only)': make_clickable_PMID}).render())\n",
    "f.close()\n",
    "display(HTML('<H3>See the complete table here (sort, filter, and search):</H3>')\n",
    "        ,FileLink('./results/PMIDs_oldVersions_only.html', result_html_prefix='<b>', result_html_suffix='</b>'))                 \n",
    "display(Markdown(\"-----------------------------------------------------\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-03T10:49:07.699796Z",
     "start_time": "2019-10-03T10:42:00.079550Z"
    },
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12954it [07:10, 30.06it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<H3>Complete Tabular Results in CSV or HTML Format:</H3>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<li><b>./results/</b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results/full_data.html' target='_blank'>full_data.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results/full_data.csv' target='_blank'>full_data.csv</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results/full_data_SPSS-friendy.csv' target='_blank'>full_data_SPSS-friendy.csv</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results/PMIDs_oldVersions_only.html' target='_blank'>PMIDs_oldVersions_only.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results/PMIDs_latestVersions_only.html' target='_blank'>PMIDs_latestVersions_only.html</a></b></li>"
      ],
      "text/plain": [
       "./results/\n",
       "  full_data.html\n",
       "  full_data.csv\n",
       "  full_data_SPSS-friendy.csv\n",
       "  PMIDs_oldVersions_only.html\n",
       "  PMIDs_latestVersions_only.html"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<H3>Specialized HTML results for each <a href=\"https://en.wikipedia.org/wiki/Wikipedia:WikiProject_Medicine\">             WikiProject medicine</a> task force              (see list of active task forces <a href=\"https://en.wikipedia.org/wiki/Wikipedia:WikiProject_Medicine/Task_forces\">             here</a>):</H3>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<li><b>./results//HTML_results_per_task_force/</b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Pathology_taskForce.html' target='_blank'>Pathology_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Nephrology_taskForce.html' target='_blank'>Nephrology_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Balkan military history_taskForce.html' target='_blank'>Balkan military history_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/European military history_taskForce.html' target='_blank'>European military history_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Applied Linguistics_taskForce.html' target='_blank'>Applied Linguistics_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Radiology_taskForce.html' target='_blank'>Radiology_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Reproductive medicine_taskForce.html' target='_blank'>Reproductive medicine_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Hematology-oncology_taskForce.html' target='_blank'>Hematology-oncology_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Gastroenterology_taskForce.html' target='_blank'>Gastroenterology_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Ethics_taskForce.html' target='_blank'>Ethics_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Neurology_taskForce.html' target='_blank'>Neurology_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Livestock_taskForce.html' target='_blank'>Livestock_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Sustainability_taskForce.html' target='_blank'>Sustainability_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Emergency medicine and EMS_taskForce.html' target='_blank'>Emergency medicine and EMS_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Psychiatry_taskForce.html' target='_blank'>Psychiatry_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Medical genetics_taskForce.html' target='_blank'>Medical genetics_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Military logistics and medicine_taskForce.html' target='_blank'>Military logistics and medicine_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/World War II_taskForce.html' target='_blank'>World War II_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Theoretical Linguistics_taskForce.html' target='_blank'>Theoretical Linguistics_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Dermatology_taskForce.html' target='_blank'>Dermatology_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Cardiology_taskForce.html' target='_blank'>Cardiology_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Ophthalmology_taskForce.html' target='_blank'>Ophthalmology_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/World War I_taskForce.html' target='_blank'>World War I_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/German military history_taskForce.html' target='_blank'>German military history_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Pulmonology_taskForce.html' target='_blank'>Pulmonology_taskForce.html</a></b></li>\n",
       "<li><b>&nbsp;&nbsp;<a href='./results//HTML_results_per_task_force/Toxicology_taskForce.html' target='_blank'>Toxicology_taskForce.html</a></b></li>"
      ],
      "text/plain": [
       "./results//HTML_results_per_task_force/\n",
       "  Pathology_taskForce.html\n",
       "  Nephrology_taskForce.html\n",
       "  Balkan military history_taskForce.html\n",
       "  European military history_taskForce.html\n",
       "  Applied Linguistics_taskForce.html\n",
       "  Radiology_taskForce.html\n",
       "  Reproductive medicine_taskForce.html\n",
       "  Hematology-oncology_taskForce.html\n",
       "  Gastroenterology_taskForce.html\n",
       "  Ethics_taskForce.html\n",
       "  Neurology_taskForce.html\n",
       "  Livestock_taskForce.html\n",
       "  Sustainability_taskForce.html\n",
       "  Emergency medicine and EMS_taskForce.html\n",
       "  Psychiatry_taskForce.html\n",
       "  Medical genetics_taskForce.html\n",
       "  Military logistics and medicine_taskForce.html\n",
       "  World War II_taskForce.html\n",
       "  Theoretical Linguistics_taskForce.html\n",
       "  Dermatology_taskForce.html\n",
       "  Cardiology_taskForce.html\n",
       "  Ophthalmology_taskForce.html\n",
       "  World War I_taskForce.html\n",
       "  German military history_taskForce.html\n",
       "  Pulmonology_taskForce.html\n",
       "  Toxicology_taskForce.html"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#------------------------------------------------------------------------------------------------------\n",
    "# tabular data vizualization, and storage in CSV and HTML format (find these files in ./results folder)\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "\n",
    "README=\"\"\"\n",
    "# WPM2Cochrane - a tool for linking WikiProject Medicine to the Cochrane Library\n",
    "\n",
    "## Launch in JupyterLab (recommended) [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/ajoorabchi/WP2Cochrane/master?urlpath=lab/tree/index.ipynb)\n",
    "\n",
    "## Launch in Jupyter Notebook  [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/ajoorabchi/WP2Cochrane/master?filepath=index.ipynb)\n",
    "\n",
    "## Results of linking (full dataset)\n",
    "- [Complete Tabular Results in HTML](https://ajoorabchi.github.io/WP2Cochrane/results/full_data.html)\n",
    "- [Complete Tabular Results in CSV](https://ajoorabchi.github.io/WP2Cochrane/results/full_data.csv)\n",
    "- [Complete Tabular Results in CSV (SPSS-friendly version)](https://ajoorabchi.github.io/WP2Cochrane/results/full_data_SPSS-friendy.csv)\n",
    "- [PMIDs statistical summary 1 (LATEST VERSIONS of Cochrane reviews found and/or cited)](https://ajoorabchi.github.io/WP2Cochrane/results/PMIDs_latestVersions_only.html)\n",
    "- [PMIDs statistical summary 2 (OLD VERSIONS of Cochrane reviews found and/or cited)](https://ajoorabchi.github.io/WP2Cochrane/results/PMIDs_oldVersions_only.html)\n",
    "\n",
    "## Results of linking (Specialized HTML results per task force)\n",
    "\"\"\"\n",
    "\n",
    "datatables_js_script=\"\"\"\n",
    "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.datatables.net/1.10.19/css/jquery.dataTables.css\">\n",
    "<script src=\"https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js\"></script>\n",
    "<script type=\"text/javascript\" charset=\"utf8\" src=\"https://cdn.datatables.net/1.10.19/js/jquery.dataTables.js\"></script>\n",
    "<script>\n",
    "$(document).ready( function () {\n",
    "    $('table.results').dataTable({\n",
    "    \"lengthMenu\": [[10, 25, 50, -1], [10, 25, 50, \"All\"]], \"bAutoWidth\": false, \n",
    "        \"columnDefs\": [\n",
    "            { \"width\": \"100%\", \"targets\": 7 }\n",
    "        ]\n",
    "    });\n",
    "    //$('table.dataframe').dataTable();    \n",
    "} );\n",
    "</script>\n",
    "\"\"\"\n",
    "\n",
    "def get_df_info(content: pd.DataFrame):\n",
    "#function code from: https://stackoverflow.com/a/44087453/2339926\n",
    "    #display(content.info())\n",
    "    content_info = StringIO()\n",
    "    content.info(buf=content_info)\n",
    "    str_ = content_info.getvalue()\n",
    "\n",
    "    lines = str_.split(\"\\n\")\n",
    "    table = StringIO(\"\\n\".join(lines[3:-3]))\n",
    "    datatypes = pd.read_csv(table, delim_whitespace=True, \n",
    "                   names=[\"column\", \"count\", \"null\", \"dtype\"])\n",
    "    datatypes.set_index(\"column\", inplace=True)\n",
    "\n",
    "    info = \"\\n\".join(lines[0:2] + lines[-2:-1])\n",
    "\n",
    "    return info, datatypes\n",
    "\n",
    "def make_clickable_wkd_items(val):\n",
    "    # target _blank to open new window\n",
    "    return '<a target=\"_blank\" href=\"{}\">{}</a>'.format(val, val[31:])\n",
    "\n",
    "def make_clickable_taskForces(val):\n",
    "    # target _blank to open new window\n",
    "    if not val: \n",
    "        return None\n",
    "    else:\n",
    "        html=\"\"\n",
    "        for taskForce in val:\n",
    "            html += f'<p><a target=\"_blank\" href=\"https://en.wikipedia.org/wiki/Wikipedia:WikiProject_Medicine/\\\n",
    "            {taskForce.replace(\" \",\"_\")}_task_force\">{taskForce}</a></p>' \n",
    "    return html\n",
    "\n",
    "def make_clickable_WP_en_articles(val):\n",
    "    # target _blank to open new window\n",
    "    if not val: \n",
    "        return None\n",
    "    else:\n",
    "        return '<a target=\"_blank\" href=\"{}\">{}</a>'.format(val, unquote(unquote(val[30:].replace(\"_\", \" \"))))\n",
    "    \n",
    "def list2text(list_):\n",
    "    txt=\"\"\n",
    "    for item in list_:\n",
    "        txt += item + \", \"\n",
    "    return txt[:-2]\n",
    "\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "# full-data.csv generator\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "h = html2text.HTML2Text()\n",
    "h.ignore_links = False\n",
    "df_plainText =df.copy()\n",
    "df_plainText['cochrane_reviews_plainText'] = [h.handle(text) if text is not None else  text for text in df['cochrane_reviews_html']]\n",
    "df_plainText['taskForces_plainText'] = [list2text(text) if text is not None else  text for text in df['taskForces']]\n",
    "df_plainText.to_csv('./results/full_data.csv', index=False)\n",
    "\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "# full_data_SPSS-friendy.csv generator (one review per row)\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "df_spss_friendly=pd.DataFrame()\n",
    "for index, row in tqdm(df[['WkD_disease', 'WkD_diseaseLabel', 'WP_en_article','class'\n",
    "                      ,'importance','taskForces','cochrane_reviews','talkPage_categories']].iterrows()):\n",
    "    #display(index,row)\n",
    "    PIMD_paper_dict = row['cochrane_reviews']\n",
    "    if PIMD_paper_dict:\n",
    "        search_results_count = len(PIMD_paper_dict)\n",
    "        cited_count=0\n",
    "        outofdate_cited_count=0\n",
    "        for PMID, paper in PIMD_paper_dict.items():\n",
    "            title = paper[0]\n",
    "            cited = paper[1]\n",
    "            latestVersion = paper[2]\n",
    "            if cited:\n",
    "                cited_count +=1\n",
    "            if cited and not latestVersion:\n",
    "                outofdate_cited_count +=1\n",
    "    else:\n",
    "        search_results_count = None\n",
    "        cited_count = None\n",
    "        outofdate_cited_count = None\n",
    "    \n",
    "    PIMD_paper_dict = row['cochrane_reviews']\n",
    "    if PIMD_paper_dict:\n",
    "        for PMID, paper in PIMD_paper_dict.items():\n",
    "            title = paper[0]\n",
    "            cited = paper[1]\n",
    "            latestVersion = paper[2]\n",
    "            data = pd.DataFrame(row.items())\n",
    "            data = data.transpose()\n",
    "            data.columns = data.iloc[0]\n",
    "            data = data.drop(data.index[[0]])\n",
    "            data = data.drop(columns=['cochrane_reviews'])\n",
    "            data['PMID']=PMID\n",
    "            data['Title']=title\n",
    "            data['Cited']=cited\n",
    "            data['LatestVersion']=latestVersion\n",
    "            data['Search results count'] = search_results_count\n",
    "            data['Cited count'] = cited_count\n",
    "            data['outofdate_cited_count'] = outofdate_cited_count\n",
    "            df_spss_friendly = df_spss_friendly.append(data)\n",
    "    else:\n",
    "            data = pd.DataFrame(row.items())\n",
    "            data = data.transpose()\n",
    "            data.columns = data.iloc[0]\n",
    "            data = data.drop(data.index[[0]])\n",
    "            data = data.drop(columns=['cochrane_reviews'])\n",
    "            data['PMID']=None\n",
    "            data['Title']=None\n",
    "            data['Cited']=None\n",
    "            data['LatestVersion']=None\n",
    "            data['Search results count'] = None\n",
    "            data['Cited count'] = None\n",
    "            data['outofdate_cited_count'] = None\n",
    "            df_spss_friendly = df_spss_friendly.append(data)\n",
    "            \n",
    "#     if index==10:\n",
    "#         break\n",
    "        \n",
    "df_spss_friendly.to_csv('./results/full_data_SPSS-friendy.csv', index=False)\n",
    "\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "# full-data.html generator\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "th    = dict(selector=\"th\",           props=[('text-align', 'left'),('font','blod 14px arial, sans-serif'),('vertical-align','top')])\n",
    "rh    = dict(selector=\".row_heading\", props=[(\"text-align\", \"left\"),('font','bold 14px arial, sans-serif'),('vertical-align','top')])\n",
    "\n",
    "col0 = dict(selector=\".col0\", props=[(\"text-align\", \"left\"),('font','bold 12px arial, sans-serif'),('max-width','100px'),('vertical-align','top')])\n",
    "col1 = dict(selector=\".col1\", props=[(\"text-align\", \"left\"),('font','bold 12px arial, sans-serif'),('max-width','130px'),('vertical-align','top')])\n",
    "col2 = dict(selector=\".col2\", props=[(\"text-align\", \"left\"),('font','bold 12px arial, sans-serif'),('max-width','110px'),('vertical-align','top'),('word-wrap','break-word')])\n",
    "col3 = dict(selector=\".col3\", props=[(\"text-align\", \"left\"),('font','bold 12px arial, sans-serif'),('max-width','60px'),('vertical-align','top'),('word-wrap','break-word')])\n",
    "col4 = dict(selector=\".col4\", props=[(\"text-align\", \"left\"),('font','bold 12px arial, sans-serif'),('max-width','90px'),('vertical-align','top'),('word-wrap','break-word')])\n",
    "col5 = dict(selector=\".col5\", props=[(\"text-align\", \"left\"),('font','bold 12px arial, sans-serif'),('max-width','90px'),('vertical-align','top'),('word-wrap','break-word')])\n",
    "\n",
    "col6 = dict(selector=\".col6\", props=[(\"text-align\", \"left\"),('font','12px arial, sans-serif'),('vertical-align','top')])\n",
    "\n",
    "fullSize =  sys.maxsize\n",
    "testSize =  10\n",
    "df_vizTable = df[['WkD_disease', 'WkD_diseaseLabel', 'WP_en_article','class','importance','taskForces','cochrane_reviews_html']]\n",
    "df_vizTable_styled=df_vizTable\\\n",
    "        .head(fullSize).rename_axis('index')\\\n",
    "        .sort_values(by=['importance'],na_position='last')\\\n",
    "        .style.set_table_styles([th,rh,col0,col1,col2,col3,col4,col5,col6]).set_table_attributes('class=\"results\"')\\\n",
    "        .format({'WkD_disease': make_clickable_wkd_items,'WP_en_article': make_clickable_WP_en_articles,\n",
    "                 'taskForces': make_clickable_taskForces})\n",
    "\n",
    "tableGuide='<H3>Table Guide:</H3>\\\n",
    "        <p><b>Grouping:</b></p>\\\n",
    "        <p>In cases, where there are multiple version of a Cochrane review, they are grouped toghther (showing the same background color)\\\n",
    "        ,and are listed chronologically,latest version first.</p>\\\n",
    "        <p><b>Color Codes:</b></p>\\\n",
    "        <ol>\\\n",
    "          <li><p style=\"color:Green;\">Green:  up-to-date and  CITED</p></li>\\\n",
    "          <li><p style=\"color:Red;\">  Red:    up-to-date and  NOT CITED</p></li>\\\n",
    "        <li><p style=\"color:Orange;\">Orange:  out-of-date and CITED</p></li>\\\n",
    "            <li><p style=\"color:Grey;\">Grey:  out-of-date and NOT CITED</p></li>\\\n",
    "        </ol><hr>'\n",
    "\n",
    "f=open(\"./results/full_data.html\",\"w\")\n",
    "info,datatypes = get_df_info(df_vizTable)\n",
    "#display(datatypes,datatypes.at['WkD_disease','count'])\n",
    "table_size_message=f\"<H3>Results Table contains <i>{datatypes.at['WkD_disease','count']}</i> rows</H3><hr>\"\n",
    "table_size_warning=f\"<H3>This is a large table, so it could take up to 30 seconds to fully load and render in your browser</H3><hr>\"\n",
    "\n",
    "if datatypes.at['WkD_disease','count']>1000:\n",
    "    table_size_message += table_size_warning\n",
    "f.write(datatables_js_script + info + datatypes.to_html() + tableGuide + table_size_message\n",
    "        + df_vizTable_styled.render())\n",
    "f.close()\n",
    "\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "# Creates a dedicated HTML rsults file for each WikiProject medicine task force group in Wikipeida\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "for taskForce, count in taskForces_count.items():\n",
    "    if taskForce!=\"NA\":\n",
    "\n",
    "#         display(HTML(f\"Creating a HTML results file for task Force <b>'{taskForce}'</b>:\\\n",
    "#                      <i>./results/HTML_results_per_task_force/{taskForce}_taskForce.html</i>\"))\n",
    "\n",
    "        #vizTable_per_taskForce = df_vizTable.loc[df_vizTable['taskForces'].apply(str)==\"[]\"]\n",
    "        vizTable_per_taskForce = df_vizTable.loc[df_vizTable['taskForces'].notnull()]\n",
    "        mask = vizTable_per_taskForce.taskForces.apply(lambda x: taskForce in x)\n",
    "        vizTable_per_taskForce = vizTable_per_taskForce[mask]\n",
    "\n",
    "       \n",
    "        vizTable_per_taskForce_styled = vizTable_per_taskForce.rename_axis('index')\\\n",
    "        .sort_values(by=['importance'],na_position='last')\\\n",
    "        .style.set_table_styles([th,rh,col0,col1,col2,col3,col4,col5,col6]).set_table_attributes('class=\"results\"')\\\n",
    "        .format({'WkD_disease': make_clickable_wkd_items,'WP_en_article': make_clickable_WP_en_articles,\n",
    "                 'taskForces': make_clickable_taskForces})\n",
    "\n",
    "        f=open(f\"./results/HTML_results_per_task_force/{taskForce}_taskForce.html\",\"w\")\n",
    "        info,datatypes = get_df_info(vizTable_per_taskForce)\n",
    "        #display(datatypes,datatypes.at['WkD_disease','count'])\n",
    "        table_size_message=f\"<H3>Results Table contains <i>{datatypes.at['WkD_disease','count']}</i> rows</H3><hr>\"\n",
    "        table_size_warning=f\"<H3>This is a large table so it needs ~10s to fully load and render in your browser</H3><hr>\"\n",
    "\n",
    "        if datatypes.at['WkD_disease','count']>1000:\n",
    "            table_size_message += table_size_warning\n",
    "        f.write(datatables_js_script + info + datatypes.to_html() + tableGuide + table_size_message\n",
    "                + vizTable_per_taskForce_styled.render()) \n",
    "        \n",
    "        f.close()\n",
    "        #update README.md\n",
    "        README += f\"\\n- [{taskForce}](https://ajoorabchi.github.io/WP2Cochrane/results/HTML_results_per_task_force/{quote(taskForce)}_taskForce.html)\"\n",
    "\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "# display CSV & HTML view/download options\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "display(HTML('<H3>Complete Tabular Results in CSV or HTML Format:</H3>'),FileLinks('./results/'\\\n",
    "            , result_html_prefix='<li><b>', result_html_suffix='</b></li>',recursive=False))\n",
    "\n",
    "display(HTML('<H3>Specialized HTML results for each <a href=\"https://en.wikipedia.org/wiki/Wikipedia:WikiProject_Medicine\">\\\n",
    "             WikiProject medicine</a> task force \\\n",
    "             (see list of active task forces <a href=\"https://en.wikipedia.org/wiki/Wikipedia:WikiProject_Medicine/Task_forces\">\\\n",
    "             here</a>):</H3>'),FileLinks('./results//HTML_results_per_task_force/'\\\n",
    "             , result_html_prefix='<li><b>', result_html_suffix='</b></li>',recursive=False))\n",
    "\n",
    "\n",
    "#display(HTML(datatables_js_script + info + datatypes.to_html() + tableGuide + df_vizTable_styled.render()))\n",
    "\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "# write README.md\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "f=open(\"./README.md\",\"w\")\n",
    "f.write(README)\n",
    "f.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-10T16:10:10.237634Z",
     "start_time": "2019-04-10T16:10:04.521317Z"
    },
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "###### ------------------------------------------------------------------------------------------------------\n",
    "# WikiData search: searches WkD for a list of dieeas\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "\"\"\"\n",
    "This is a modified version of code from: \n",
    "1. https://lawlesst.github.io/notebook/sparql-dataframe.html\n",
    "2. https://github.com/SuLab/sparql_to_pandas/blob/master/SPARQL_pandas.ipynb\n",
    "\n",
    "Demonstrating how to get JupyterLab working with Binder: \n",
    "\n",
    "https://github.com/binder-examples/jupyterlab\n",
    "https://github.com/binder-examples/jupyter-extension/blob/master/index.ipynb\n",
    "\"\"\"\n",
    "\n",
    "def get_sparql_dataframe(service, query):\n",
    "    \"\"\"\n",
    "    Helper function to convert SPARQL results into a Pandas data frame.\n",
    "    \"\"\"\n",
    "    sparql = SPARQLWrapper(service)\n",
    "    sparql.setQuery(query)\n",
    "    sparql.setReturnFormat(JSON)\n",
    "    result = sparql.query()\n",
    "\n",
    "    processed_results = json.load(result.response)\n",
    "    cols = processed_results['head']['vars']\n",
    "\n",
    "    out = []\n",
    "    for row in processed_results['results']['bindings']:\n",
    "        item = []\n",
    "        for c in cols:\n",
    "            item.append(row.get(c, {}).get('value'))\n",
    "        out.append(item)\n",
    "\n",
    "    return pd.DataFrame(out, columns=cols)\n",
    "\n",
    "#help: https://en.wikibooks.org/wiki/SPARQL/Wikidata_Query_Service_-_Introduction\n",
    "\n",
    "wds = \"https://query.wikidata.org/sparql\"\n",
    "rq = \"\"\"\n",
    "SELECT ?WkD_disease ?WkD_diseaseLabel ?WP_en_article \n",
    "WHERE {\n",
    "  SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\". }\n",
    "  ?WkD_disease wdt:P31 wd:Q12136.\n",
    "  \n",
    "      OPTIONAL {\n",
    "      ?WP_en_article schema:about ?WkD_disease .\n",
    "      ?WP_en_article schema:inLanguage \"en\" .\n",
    "      ?WP_en_article schema:isPartOf <https://en.wikipedia.org/> .\n",
    "    }\n",
    "}\n",
    "#order by desc(?WkD_disease)\n",
    "\"\"\"\n",
    "\n",
    "df = get_sparql_dataframe(wds, rq)\n",
    "\n",
    "#WkD api sample\n",
    "# client = Client()  # doctest: +SKIP\n",
    "# entity = client.get('Q1472', load=True)\n",
    "# print (entity)\n",
    "# print (entity.description)\n",
    "\n",
    "# image_prop = client.get('P18')\n",
    "# image = entity[image_prop]\n",
    "# print (image)\n",
    "\n",
    "# print(image.image_resolution)\n",
    "\n",
    "# print(image.image_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-03T10:49:07.709745Z",
     "start_time": "2019-10-03T10:49:07.701463Z"
    },
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "#------------------------------------------------------------------------------------------------------\n",
    "# PubMed search \n",
    "#------------------------------------------------------------------------------------------------------\n",
    "\n",
    "\"\"\"\n",
    "This is a modified version of code from: \n",
    "1. https://gist.github.com/bonzanini/5a4c39e4c02502a8451d\n",
    "2. https://gist.github.com/bonzanini/5a4c39e4c02502a8451d\n",
    "\n",
    "# Full discussion:\n",
    "# https://marcobonzanini.wordpress.com/2015/01/12/searching-pubmed-with-python/\n",
    "\"\"\"\n",
    "\n",
    "def search(index,query):\n",
    "    Entrez.email = 'your.email@example.com'\n",
    "    \n",
    "    logger.debug (f\"{index:>5}. PubMed search query: {query}\")\n",
    "    handle = Entrez.esearch(db='pubmed', sort='relevance', retmax='200', retmode='xml', term=query)\n",
    "    results = Entrez.read(handle)\n",
    "    return results\n",
    "\n",
    "def fetch_details(index, id_list):\n",
    "    ids = ','.join(id_list)\n",
    "    Entrez.email = 'your.email@example.com'\n",
    "\n",
    "    sleep_time = 10\n",
    "    num_retries = 1000\n",
    "    error = None\n",
    "    for x in range(0, num_retries):  \n",
    "        try:\n",
    "            logger.debug (f\"{index:>5}. Fetching article details for PMIDs: {ids}\")\n",
    "            handle  = Entrez.efetch(db='pubmed', retmode='xml', id=ids)\n",
    "            results = Entrez.read(handle)\n",
    "            error = None\n",
    "        except Exception as error:\n",
    "            logger.error (f\"{index:>5}. I got a HTTPError - reason {error}- while trying to fetch the articles details from pubmed\")\n",
    "            logger.error (f\"{index:>5}. Sleeping for {sleep_time} seconds before trying again...\")\n",
    "            sleep(sleep_time)  # wait before trying to fetch the data again\n",
    "            sleep_time *= 1.5  # Implement your backoff algorithm here i.e. exponential backoff\n",
    "            pass\n",
    "\n",
    "        if not error:\n",
    "            break\n",
    "            \n",
    "    if error:\n",
    "        logger.critical (f\"{index:>5}. Retried fetching article's details {num_retries} times with no success!\")\n",
    "        raise error\n",
    "    return results\n",
    "\n",
    "def pubmed_search(index, disease, searchTitle, searchAbstract):\n",
    "    \n",
    "    logger.debug (f\"{index:>5}. Searching PubMed for {disease} in searchTitle:{searchTitle} , searchAbstract:{searchAbstract}\")\n",
    "    if searchTitle and searchAbstract:\n",
    "        results = search(index,\"(\"+disease+'[Title/Abstract] NOT \"withdrawn\"[Title]) AND \"The Cochrane database of systematic reviews\"[Journal]')\n",
    "    if searchTitle and not searchAbstract:\n",
    "        results = search(index,\"(\"+disease+'[Title] NOT \"withdrawn\"[Title]) AND \"The Cochrane database of systematic reviews\"[Journal]')\n",
    "        \n",
    "    #pprint.pprint (results)\n",
    "    return results['IdList']\n",
    "\n",
    "# display(len(pubmed_search(\"Crohn's disease\",True,True)))\n",
    "# display(len(pubmed_search(\"Crohn's disease\",True,False)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-03T10:49:07.724984Z",
     "start_time": "2019-10-03T10:49:07.713900Z"
    },
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "#------------------------------------------------------------------------------------------------------\n",
    "# string comparator \n",
    "#------------------------------------------------------------------------------------------------------\n",
    "\n",
    "# def searh_wp_refs_4title(pubmedArticleTitle,WPpageTitle):\n",
    "#     ''' looksup an article title in a WikiPedia page '''\n",
    "#     #display (wikipedia.WikipediaPage(WPpageTitle).html())\n",
    "#     WPpageHTML= wikipedia.WikipediaPage(WPpageTitle).html()\n",
    "#     ratio = fuzz.ratio(WPpageHTML, pubmedArticleTitle)\n",
    "#     print (\"fuzz.ratio:\",ratio)\n",
    "\n",
    "#     WPpageHTML=re.sub(r'\\W+', '', WPpageHTML.lower())\n",
    "#     pubmedArticleTitle=re.sub(r'\\W+', '', pubmedArticleTitle.lower())\n",
    "#     print(pubmedArticleTitle)\n",
    "#     if pubmedArticleTitle in WPpageHTML:\n",
    "#         return True\n",
    "#     else:\n",
    "#         return False\n",
    "    \n",
    "#print (searh_wp_refs_4title(\"interventions to slow progression of myopia in children...\",\"Near-sightedness\"))\n",
    "\n",
    "def searh_wp_refs_4PMID(PMID,en_article_HTML):\n",
    "    ''' looksup a PMIDs in a WikiPedia page '''  \n",
    "\n",
    "    #print('searching for PMID: <b>'+PMID+ '</b>')\n",
    "    if PMID in en_article_HTML:\n",
    "        #print ('found')\n",
    "        return True\n",
    "    else:\n",
    "        #print ('not found')\n",
    "        return False\n",
    "    \n",
    "#en_article_HTML= wikipedia.WikipediaPage('Near-sightedness').html()\n",
    "#print (searh_wp_refs_4PMID(\"22161388\",en_article_HTML))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-03T10:53:57.254225Z",
     "start_time": "2019-10-03T10:49:07.728865Z"
    },
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "454a55c1d4d64145975defe1582a0cfd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6905cf6f0e724e90851efabe7cff430c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "937316ce9b824c2b9886e6a3ebc8e0f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Progress', max=4717, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    376\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 2.7, use buffering of HTTP responses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 377\u001b[0;31m                 \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffering\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    378\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: getresponse() got an unexpected keyword argument 'buffering'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-e0f69c66ef4e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     45\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{index:>5}. Getting the WikiPedia content for: {WP_en_article_Title}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mWP_en_article_obj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwikipedia\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mWikipediaPage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mWP_en_article_Title\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m             \u001b[0mWP_en_article_HTML\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mWP_en_article_obj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhtml\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m             \u001b[0;31m#checking for redirects\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/wikipedia/wikipedia.py\u001b[0m in \u001b[0;36mhtml\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    452\u001b[0m       }\n\u001b[1;32m    453\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 454\u001b[0;31m       \u001b[0mrequest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_wiki_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    455\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_html\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'query'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'pages'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpageid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'revisions'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'*'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/wikipedia/wikipedia.py\u001b[0m in \u001b[0;36m_wiki_request\u001b[0;34m(params)\u001b[0m\n\u001b[1;32m    735\u001b[0m     \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait_time\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal_seconds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    736\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 737\u001b[0;31m   \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mAPI_URL\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    738\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    739\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mRATE_LIMIT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/requests/api.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     73\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m     \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'allow_redirects'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 75\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'get'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     76\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/requests/api.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    531\u001b[0m         }\n\u001b[1;32m    532\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 533\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    534\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    535\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    667\u001b[0m         \u001b[0;31m# Resolve redirects if allowed.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 668\u001b[0;31m         \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mresp\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mresp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgen\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mallow_redirects\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    669\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    670\u001b[0m         \u001b[0;31m# Shuffle things around if there's history.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    667\u001b[0m         \u001b[0;31m# Resolve redirects if allowed.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 668\u001b[0;31m         \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mresp\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mresp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgen\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mallow_redirects\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    669\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    670\u001b[0m         \u001b[0;31m# Shuffle things around if there's history.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mresolve_redirects\u001b[0;34m(self, resp, req, stream, timeout, verify, cert, proxies, yield_requests, **adapter_kwargs)\u001b[0m\n\u001b[1;32m    245\u001b[0m                     \u001b[0mproxies\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mproxies\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    246\u001b[0m                     \u001b[0mallow_redirects\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 247\u001b[0;31m                     \u001b[0;34m**\u001b[0m\u001b[0madapter_kwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    248\u001b[0m                 )\n\u001b[1;32m    249\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    644\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    645\u001b[0m         \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 646\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    647\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    648\u001b[0m         \u001b[0;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    447\u001b[0m                     \u001b[0mdecode_content\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m                     \u001b[0mretries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m                     \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    450\u001b[0m                 )\n\u001b[1;32m    451\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    598\u001b[0m                                                   \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout_obj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    599\u001b[0m                                                   \u001b[0mbody\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 600\u001b[0;31m                                                   chunked=chunked)\n\u001b[0m\u001b[1;32m    601\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    602\u001b[0m             \u001b[0;31m# If we're going to release the connection in ``finally:``, then\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    378\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    379\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 380\u001b[0;31m                     \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    381\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    382\u001b[0m                     \u001b[0;31m# Remove the TypeError from the exception chain in Python 3;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mgetresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1329\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1330\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1331\u001b[0;31m                 \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1332\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1333\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mbegin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    295\u001b[0m         \u001b[0;31m# read until we get a non-100 response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    296\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 297\u001b[0;31m             \u001b[0mversion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreason\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    298\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mCONTINUE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36m_read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    256\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 258\u001b[0;31m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"iso-8859-1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    259\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mLineTooLong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"status line\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    584\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    585\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    292\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    293\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 294\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    295\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mOpenSSL\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSSL\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSysCallError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    296\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msuppress_ragged_eofs\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Unexpected EOF'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/OpenSSL/SSL.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1819\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_lib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSSL_peek\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ssl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnbytes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1820\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1821\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_lib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSSL_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ssl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnbytes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1822\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_raise_ssl_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ssl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1823\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\"\"\"------------------------------------------------------------------------------------------------------\n",
    "Main: this is the main part of the code which encompasses the following steps:\n",
    "\n",
    "    1. content and metadata of WP articles which correspond to a disease in WikiData \n",
    "    are retrieved (title, class, importance, taskforce, content)\n",
    "\n",
    "    2. For each WP article a search query is submitted to PubMed to retrieve a list of relevant Cochrane reviews\n",
    "\n",
    "    3. the retrieved list of reviews are cross checked agains the WP articles reference section to identify cited, missing\n",
    "    and out of date reviews.\n",
    "\n",
    "    4. gathered infomation is converted tocolor-coded tabular format.\n",
    "------------------------------------------------------------------------------------------------------\"\"\"\n",
    "\n",
    "h2t = html2text.HTML2Text()\n",
    "h2t.ignore_links = True\n",
    "\n",
    "df['class']=None\n",
    "df['importance']=None\n",
    "df['taskForces']=None\n",
    "df['cochrane_reviews_html']=None\n",
    "df['cochrane_reviews']=None\n",
    "df['talkPage_categories']=None\n",
    "\n",
    "disease_search_log= widgets.HTML()\n",
    "citation_search_log= widgets.HTML()\n",
    "\n",
    "display(disease_search_log)\n",
    "display(citation_search_log)\n",
    "for index, row in tqdm_notebook(df.iterrows(),desc='Progress',total=df['WP_en_article'].count(), unit=\"WkD_disease\"):\n",
    "    if row['WP_en_article'] is not None:\n",
    "        #and row['WP_en_article']==\"https://en.wikipedia.org/wiki/Agalactia\":\n",
    "        if console_handler.level<=20:\n",
    "            display(HTML(f\"<b>START of row {index:>5}<b>\"))\n",
    "        else:\n",
    "            logger.info(f\"<{index:>5}------------------------------------------------\")\n",
    "\n",
    "        disease_search_log.value= f'<p style=\"color:blue;\"><b>processing:</b> disease #{index:>5}\\\n",
    "            &emsp; {row[\"WkD_diseaseLabel\"]} &emsp; {row[\"WkD_disease\"]} &emsp; {row[\"WP_en_article\"]} </p>'\n",
    "        WP_en_article_Title= unquote(unquote(row['WP_en_article'][30:]))\n",
    "        \n",
    "        #will raise a DisambiguationError if the page is a disambiguation page, or a PageError if \n",
    "        #the page doesn’t exist (although by default, it tries to find the page you meant with suggest and search.)\n",
    "        try:\n",
    "            logger.info(f\"{index:>5}. Getting the WikiPedia content for: {WP_en_article_Title}\")\n",
    "            WP_en_article_obj = wikipedia.WikipediaPage(WP_en_article_Title)\n",
    "            WP_en_article_HTML = WP_en_article_obj.html()\n",
    "            \n",
    "            #checking for redirects\n",
    "            if WP_en_article_Title.casefold() != WP_en_article_obj.title.casefold().replace(\" \",\"_\"):\n",
    "                logger.info(f\"{index:>5}. Redirected to: {WP_en_article_obj.title}\")            \n",
    "            \n",
    "            logger.info(f\"{index:>5}. Getting the WikiPedia talkPage_categories for: {WP_en_article_obj.title}\")\n",
    "            WP_en_article_talk_obj = wikipedia.WikipediaPage(\"Talk:\"+ WP_en_article_obj.title)\n",
    "            row['talkPage_categories'] = WP_en_article_talk_obj.categories\n",
    "        \n",
    "        except wikipedia.exceptions.DisambiguationError as e:\n",
    "            logger.warning (f\"{index:>5}. '{WP_en_article_talk_obj.title}' is an Ambiguous title: {e.options}\")\n",
    "            logger.warning (f\"{index:>5}. Picking the first sense in the list by default: {e.options[0]}\")\n",
    "            WP_en_article_obj = wikipedia.WikipediaPage(e.options[0])\n",
    "            WP_en_article_HTML = WP_en_article_obj.html()\n",
    "            \n",
    "            #checking for redirects\n",
    "            if e.options[0].casefold() != WP_en_article_obj.title.casefold().replace(\" \",\"_\"):\n",
    "                logger.warning (f\"{index:>5}. Redirected to: {WP_en_article_obj.title}\")\n",
    "            \n",
    "            logger.warning (f\"{index:>5}. Getting the talkPage_categories for: {WP_en_article_obj.title}\")\n",
    "            WP_en_article_talk_obj = wikipedia.WikipediaPage(\"Talk:\"+ WP_en_article_obj.title)\n",
    "            row['talkPage_categories'] = WP_en_article_talk_obj.categories\n",
    "\n",
    "        except wikipedia.exceptions.PageError as e:\n",
    "            logger.error (f\"{index:>5}. {WP_en_article_obj.title} - I got a PageError - reason: {e} - Article has no talk page yet\")\n",
    "        except KeyError as e:\n",
    "            logger.error (f\"{index:>5}. {WP_en_article_obj.title} - I got a KeyError  - reason: {e} - Article's Talk page has no Category\")\n",
    "            \n",
    "        finally:\n",
    "            if console_handler.level==10: \n",
    "                display(row['talkPage_categories'])\n",
    "            else:\n",
    "                logger.debug(row['talkPage_categories'])\n",
    "            \n",
    "        logger.debug(f\"{index:>5}. Extracting task forces, class, and importance\")\n",
    "        taskForces=[]\n",
    "        for cat in row['talkPage_categories'] or []:\n",
    "            if cat.casefold().endswith(\"-class medicine articles\"):\n",
    "                row['class'] = cat[0:-24]\n",
    "            if cat.casefold().endswith('-importance medicine articles'):\n",
    "                imp = {\n",
    "                    'NA'      : None,\n",
    "                    '???'     : None,\n",
    "                    'Unknown' : None,\n",
    "                    'Low'     : '4-Low',\n",
    "                    'Mid'     : '3-Mid',\n",
    "                    'High'    : '2-High',\n",
    "                    'Top'     : '1-Top'\n",
    "                    }\n",
    "                if imp[cat[0:-29]]:\n",
    "                    row['importance'] = imp[cat[0:-29]]\n",
    "                else:\n",
    "                    row['importance'] = None\n",
    "                \n",
    "            if cat.casefold().endswith(' task force articles') and \"wikiproject\" not in cat.casefold():\n",
    "                taskForce = cat[0:-20]\n",
    "                taskForces.append(taskForce)\n",
    "            \n",
    "        if taskForces:\n",
    "            row['taskForces']=taskForces\n",
    "        if console_handler.level<=20:\n",
    "            display(HTML(f\"{index:>5}. class: {row['class']}, importance: {row['importance']}, task forces: {row['taskForces']}\"))\n",
    "        else:\n",
    "            logger.info(f\"{index:>5}. class: {row['class']}, importance: {row['importance']}, task forces: {row['taskForces']}\")\n",
    "\n",
    "\n",
    "\n",
    "        matches=0\n",
    "        PIMD_paper_dict={}\n",
    "        logger.info(f\"{index:>5}. searching Pubmed for WkD_diseaseLabel: {row['WkD_diseaseLabel']}\") \n",
    "        id_list= pubmed_search(index,row['WkD_diseaseLabel'],True,True)\n",
    "        if len(id_list)==200:\n",
    "            logger.warning(f\"{index:>5}. (a) {row['WkD_diseaseLabel']} - Too many matches found (>200)\") \n",
    "            logger.warning(f\"{index:>5}. (b) restrcting search to Titles only (excluding Abstracts)\")\n",
    "            time.sleep(1)\n",
    "            id_list= pubmed_search(index,row['WkD_diseaseLabel'],True,False)\n",
    "            if len(id_list)==0:\n",
    "                logger.warning(f\"{index:>5}. (a) {row['WkD_diseaseLabel']} - Restrcting search to Titles only returned no results\")\n",
    "                logger.warning(f\"{index:>5}. (b) reverting back to title/abstract\")\n",
    "                time.sleep(1)\n",
    "                id_list= pubmed_search(index,row['WkD_diseaseLabel'],True,True)                      \n",
    "        \n",
    "        if not id_list and row['WkD_diseaseLabel'].lower()!= re.sub(r\" ?\\([^)]+\\)\", \"\", WP_en_article_Title.replace(\"_\", \" \").lower()):\n",
    "            logger.warning(f\"{index:>5}. (a) searching for '{row['WkD_diseaseLabel']}' returned {len(id_list)} results\")\n",
    "            logger.warning(f\"{index:>5}. (b) searching for '{WP_en_article_Title.replace('_', ' ')}' instead\")\n",
    "            time.sleep(1)\n",
    "            id_list= pubmed_search(index,WP_en_article_Title.replace(\"_\", \" \"),True,True)\n",
    "            if len(id_list)==200:\n",
    "                logger.warning(f\"{index:>5}. (a) {WP_en_article_Title.replace('_',' ')} - Retruned too many matches (>200)\")\n",
    "                logger.warning(f\"{index:>5}. (b) restrcting search to Titles only (excluding Abstracts)\")\n",
    "                time.sleep(1)\n",
    "                id_list= pubmed_search(index,WP_en_article_Title.replace(\"_\", \" \"),True,False)\n",
    "                if len(id_list)==0:\n",
    "                    logger.warning(f\"{index:>5}. (a) {WP_en_article_Title.replace('_',' ')} - Restrcting search to Titles only returned no results\")\n",
    "                    logger.warning(f\"{index:>5}. (b) reverting back to title/abstract\")\n",
    "                    time.sleep(1)\n",
    "                    id_list= pubmed_search(index,WP_en_article_Title.replace(\"_\", \" \"),True,True)                      \n",
    "        \n",
    "        logger.info(f\"{index:>5}. {len(id_list)} matching PMIDs found.\")\n",
    "        if  id_list:\n",
    "            papers = fetch_details(index,id_list)\n",
    "            #print(json.dumps(papers, indent=5))\n",
    "\n",
    "            for i, paper in enumerate(papers['PubmedArticle']):\n",
    "                articleTitle = paper['MedlineCitation']['Article']['ArticleTitle']\n",
    "                PMID = int(paper['MedlineCitation']['PMID'])\n",
    "                cited = searh_wp_refs_4PMID(str(PMID),WP_en_article_HTML)\n",
    "                if cited:\n",
    "                    matches +=1\n",
    "                #display(articleTitle,PMID,cited)\n",
    "                latestVersion=None                       \n",
    "                PIMD_paper_dict[int(PMID)]=[articleTitle,cited,latestVersion]\n",
    "                \n",
    "        PIMD_paper_dict = OrderedDict(sorted(PIMD_paper_dict.items(), key=lambda t: [str(title).lower() for title in t[1][0]]))\n",
    "        #display(PIMD_paper_dict)\n",
    "        row['cochrane_reviews']= PIMD_paper_dict\n",
    "\n",
    "        if PIMD_paper_dict:\n",
    "            row['cochrane_reviews_html']='<div align=\"left\" style=\"margin:0px;\"><ol start=\"1\" style=\"margin-left:0px\">'\n",
    "            bgc=\"White\"\n",
    "            for PMID, paper in PIMD_paper_dict.items():\n",
    "                cited=paper[1]\n",
    "                if cited:\n",
    "                    color=\"green\"\n",
    "                    cited_message=\"<b> [CITED] </b>\"\n",
    "                if not cited:\n",
    "                    color=\"red\"\n",
    "                    cited_message=\"<b> [NOT CITED] </b>\"\n",
    "                \n",
    "                title=paper[0]\n",
    "                latestVersion=True\n",
    "                foundItems = (key for key, vals in PIMD_paper_dict.items() if title.lower() in [str(val).lower() for val in vals] and key!=PMID)\n",
    "                for item in foundItems:\n",
    "                    #display(item)\n",
    "                    if item > PMID:\n",
    "                        latestVersion=False\n",
    "                paper[2]=latestVersion\n",
    "                #display(title,PMID,cited,latestVersion)\n",
    "                \n",
    "                if latestVersion:\n",
    "                    version_message=\"<b> [LATEST Version] </b>\"\n",
    "                else:\n",
    "                    version_message=\"<b> [OLD Version] </b>\"\n",
    "                    if cited:\n",
    "                        color=\"orange\"\n",
    "                        version_message +=\"<b> [UPDATE NEEDED] </b>\"\n",
    "                    else:\n",
    "                        color=\"grey\"\n",
    "                        \n",
    "                if latestVersion:\n",
    "                    if bgc==\"white\":\n",
    "                        bgc=\"#E0F5FE\"\n",
    "                    else:\n",
    "                        bgc=\"white\"\n",
    "                \n",
    "                row['cochrane_reviews_html']+='<li style=\"padding:5px;color:'+color+';background-color:'+bgc+';\">'\\\n",
    "                    +paper[0]+' <a target=\"_blank\" href=\"https://www.ncbi.nlm.nih.gov/pubmed/'\\\n",
    "                    +str(PMID)+'\">PMID: '+str(PMID)+'</a>'+cited_message+version_message+\"</li>\"\n",
    "\n",
    "            row['cochrane_reviews_html']+=\"</ol></div>\"\n",
    "        else:\n",
    "            row['cochrane_reviews_html']=\"No matching publication found!\"\n",
    "\n",
    "        \n",
    "        if console_handler.level<=10: #10=DEBUG\n",
    "            display(HTML(row['cochrane_reviews_html']))\n",
    "        else:\n",
    "            #logger.debug(h2t.handle(str(row['cochrane_reviews_html'])))\n",
    "            logger.debug(row['cochrane_reviews_html'])            \n",
    "\n",
    "        citation_search_log.value = f'<p style=\"color:green;\"><b>processed</b>: disease #{index:>5} &emsp; \\\n",
    "            {row[\"WkD_diseaseLabel\"]} &emsp; {row[\"WkD_disease\"]} &emsp; {row[\"WP_en_article\"]}</p> \\\n",
    "            <p style=\"color:green;\"><b>{matches} of {len(id_list)}</b>\\\n",
    "            Cochrane reviews found (via PubMed) are cited in the Wikipedia article: {WP_en_article_Title}</p>'\n",
    "        \n",
    "        if console_handler.level<=20:\n",
    "            display(HTML(f\"<b>END of row {index:>5}<b><hr>\"))\n",
    "        else:\n",
    "            logger.info(f\"------------------------------------------------{index:>5}><hr>\")\n",
    "                                   \n",
    "                                   \n",
    "\n",
    "\"\"\"------------------------------------------------------------------------------------------------------\n",
    "# Once all the tasks (above cells) are completed successfuly, the dataframe \n",
    "# is sotored in persistent storage for future use (e.g., Binder)\n",
    "#------------------------------------------------------------------------------------------------------\"\"\"\n",
    "# Creating a datetime object\n",
    "current_datetime = datetime.now()\n",
    "# Converting a to string in the desired format (YYYYMMDD) using strftime\n",
    "# and then to int.\n",
    "current_datetime_int = int(current_datetime.strftime('%Y%m%d'))\n",
    "\n",
    "display(HTML(f\"<h2>Saving df in ./persistent_storage/{current_datetime_int}.pkl\"))\n",
    "logger.info(f\"Saving df in ./persistent_storage/{current_datetime_int}.pkl\")\n",
    "# with open('./persistent_storage/df.dill', 'wb') as out_strm: \n",
    "#     dill.dump(df, out_strm) \n",
    "#dill.dump_session('./persistent_storage/dill_session')   \n",
    "df.to_pickle(f\"./persistent_storage/{current_datetime_int}.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "notify_time": "30",
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "157px",
    "width": "360px"
   },
   "number_sections": false,
   "sideBar": false,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": false,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "02fcf423e13c405d90881f77fd72a2f1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": "initial"
      }
     },
     "234bc1990c3248f1acdaa5e1c2e01a8c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_7a8dc09eaf464dd597c1dd4141a8b9a3",
       "style": "IPY_MODEL_25a2f984db404154b59d9a6ff1e8c508",
       "value": "| 11621/? [3:34:57&lt;00:00,  1.11s/WkD_disease]"
      }
     },
     "25a2f984db404154b59d9a6ff1e8c508": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "3e036a8f5b654e3ebd45c34a9bfcf9d4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_6f716b54558f4deb9983f274991c97d5",
       "style": "IPY_MODEL_4b5fada894fd4fe48f1f247b461db333",
       "value": "<p style=\"color:green;\"><b>WkDdiseases_with_WParticles_count</b>:             4381, &emsp; <b>total_CochraneReviews_found</b>:             19962, &emsp; <b>total_CochraneReviews_cited</b>: 1049</p>"
      }
     },
     "4b5fada894fd4fe48f1f247b461db333": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "4cd5947e74ca455c9eeb24470e34ff62": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "6c837377612a4cf5b13dbb635862403f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "6f716b54558f4deb9983f274991c97d5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "79f2b78f122846988434c41a2ad4a2d8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "7a8dc09eaf464dd597c1dd4141a8b9a3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "83d507f804354506a1f32bc53d56d18a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_a93d0d8d295e4169941c52d9363e93d5",
        "IPY_MODEL_234bc1990c3248f1acdaa5e1c2e01a8c"
       ],
       "layout": "IPY_MODEL_6c837377612a4cf5b13dbb635862403f"
      }
     },
     "8613affbed9a47ea885f3a9f79988f4a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "a93d0d8d295e4169941c52d9363e93d5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "bar_style": "success",
       "description": "Progress",
       "layout": "IPY_MODEL_ce9f975bbb6847379526446159cd96c1",
       "max": 4381,
       "style": "IPY_MODEL_02fcf423e13c405d90881f77fd72a2f1",
       "value": 4381
      }
     },
     "ce9f975bbb6847379526446159cd96c1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "d3d4db35ac0f4b1e9b3af80c0a9e63a2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_4cd5947e74ca455c9eeb24470e34ff62",
       "style": "IPY_MODEL_8613affbed9a47ea885f3a9f79988f4a",
       "value": "<p style=\"color:blue;\"><b>processing:</b> disease #11580            &emsp; Ebola virus disease &emsp; http://www.wikidata.org/entity/Q51993 &emsp; https://en.wikipedia.org/wiki/Ebola_virus_disease </p>"
      }
     },
     "e01f923030e149898ee50e28e4084a44": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "e84032ee57734e71aee666a933508946": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_79f2b78f122846988434c41a2ad4a2d8",
       "style": "IPY_MODEL_e01f923030e149898ee50e28e4084a44",
       "value": "<p style=\"color:green;\"><b>processed</b>: disease #11580 &emsp;             Ebola virus disease &emsp; http://www.wikidata.org/entity/Q51993 &emsp; https://en.wikipedia.org/wiki/Ebola_virus_disease</p>             <p style=\"color:green;\"><b>0 of 2</b>            Cochrane reviews found (via PubMed) are cited in the Wikipedia article: Ebola_virus_disease</p>"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}