dataapps/webcrawl.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Webcrawl\n",
    "ref: https://github.com/openai/openai-cookbook/blob/main/apps/web-crawl-q-and-a/web-qa.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lkk/miniconda3/envs/mypy310/lib/python3.10/site-packages/requests/__init__.py:102: RequestsDependencyWarning: urllib3 (1.26.7) or chardet (5.2.0)/charset_normalizer (2.0.7) doesn't match a supported version!\n",
      "  warnings.warn(\"urllib3 ({}) or chardet ({})/charset_normalizer ({}) doesn't match a supported \"\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import re\n",
    "import urllib.request\n",
    "from bs4 import BeautifulSoup\n",
    "from collections import deque\n",
    "from html.parser import HTMLParser\n",
    "from urllib.parse import urlparse\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Regex pattern to match a URL\n",
    "HTTP_URL_PATTERN = r'^http[s]*://.+'\n",
    "\n",
    "# Define root domain to crawl\n",
    "domain = \"sjsu.edu\"\n",
    "full_url = \"https://www.sjsu.edu/cmpe/\"\n",
    "subdomain = \"/cmpe\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ParseResult(scheme='https', netloc='www.sjsu.edu', path='/cmpe/research/tdd.pdf', params='', query='', fragment='')"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_obj = urlparse(\"https://www.sjsu.edu/cmpe/research/tdd.pdf\")\n",
    "url_obj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_obj.path.startswith(\"/cmpe\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_obj.path.endswith(\".pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a class to parse the HTML and get the hyperlinks\n",
    "class HyperlinkParser(HTMLParser):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        # Create a list to store the hyperlinks\n",
    "        self.hyperlinks = []\n",
    "\n",
    "    # Override the HTMLParser's handle_starttag method to get the hyperlinks\n",
    "    def handle_starttag(self, tag, attrs):\n",
    "        attrs = dict(attrs)\n",
    "\n",
    "        # If the tag is an anchor tag and it has an href attribute, add the href attribute to the list of hyperlinks\n",
    "        #The <a> tag defines a hyperlink\n",
    "        if tag == \"a\" and \"href\" in attrs:\n",
    "            self.hyperlinks.append(attrs[\"href\"])\n",
    "\n",
    "#takes a URL as an argument, opens the URL, and reads the HTML content. Then, it returns all the hyperlinks found on that page.\n",
    "# Function to get the hyperlinks from a URL\n",
    "def get_hyperlinks(url):\n",
    "    \n",
    "    # Try to open the URL and read the HTML\n",
    "    try:\n",
    "        # Open the URL and read the HTML\n",
    "        with urllib.request.urlopen(url) as response:\n",
    "\n",
    "            # If the response is not HTML, return an empty list\n",
    "            if not response.info().get('Content-Type').startswith(\"text/html\"):\n",
    "                return []\n",
    "            \n",
    "            # Decode the HTML\n",
    "            html = response.read().decode('utf-8')\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        return []\n",
    "\n",
    "    # Create the HTML Parser and then Parse the HTML to get hyperlinks\n",
    "    parser = HyperlinkParser()\n",
    "    parser.feed(html)\n",
    "\n",
    "    return parser.hyperlinks\n",
    "\n",
    "# Function to get the hyperlinks from a URL that are within the same domain\n",
    "def get_domain_hyperlinks(local_domain, url, subdomain=\"\"):\n",
    "    clean_links = []\n",
    "    for link in set(get_hyperlinks(url)):\n",
    "        clean_link = None\n",
    "\n",
    "        # If the link is a URL, check if it is within the same domain\n",
    "        if re.search(HTTP_URL_PATTERN, link):\n",
    "            # Parse the URL and check if the domain is the same\n",
    "            url_obj = urlparse(link)\n",
    "            if url_obj.netloc == local_domain:\n",
    "                if subdomain:\n",
    "                    if url_obj.path.startswith(subdomain):\n",
    "                        #print(\"match subdomain\")\n",
    "                        clean_link = link\n",
    "                else:\n",
    "                    clean_link = link\n",
    "\n",
    "        # If the link is not a URL, check if it is a relative link\n",
    "        else:\n",
    "            #print(\"relative link\", link)\n",
    "            if subdomain:\n",
    "                if not link.startswith(subdomain):\n",
    "                    continue\n",
    "            if link.startswith(\"/\"):\n",
    "                link = link[1:]\n",
    "            elif link.startswith(\"#\") or link.startswith(\"mailto:\"):\n",
    "                continue\n",
    "            clean_link = \"https://\" + local_domain + \"/\" + link\n",
    "\n",
    "        if clean_link is not None:\n",
    "            if clean_link.endswith(\"/\"):\n",
    "                clean_link = clean_link[:-1]\n",
    "            if clean_link.endswith(\".pdf\") or clean_link.endswith(\".jpg\") or clean_link.endswith(\".png\"):\n",
    "                 continue\n",
    "            clean_links.append(clean_link)\n",
    "\n",
    "    # Return the list of hyperlinks that are within the same domain\n",
    "    return list(set(clean_links))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/lkk/Developer/DeepDataMiningLearning/dataapps'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./output/text/'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputpath=\"./output\"\n",
    "os.mkdir(outputpath)\n",
    "textoutputpath=os.path.join(outputpath,\"text/\")\n",
    "textoutputpath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./output/text/www.sjsu.edu/'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textoutputpath+local_domain+\"/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "local_domain = urlparse(full_url).netloc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'www.sjsu.edu'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "local_domain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#crawl the given url and output text files in textoutputpath\n",
    "def crawl(url, textoutputpath, subdomain=\"\"):\n",
    "    # Parse the URL and get the domain\n",
    "    local_domain = urlparse(url).netloc\n",
    "\n",
    "    # Create a queue to store the URLs to crawl\n",
    "    queue = deque([url])\n",
    "\n",
    "    # Create a set to store the URLs that have already been seen (no duplicates)\n",
    "    seen = set([url])\n",
    "\n",
    "    # Create a directory to store the text files\n",
    "    if not os.path.exists(textoutputpath):\n",
    "            os.mkdir(textoutputpath)\n",
    "\n",
    "    if not os.path.exists(textoutputpath+local_domain+\"/\"):\n",
    "            os.mkdir(textoutputpath + local_domain + \"/\")\n",
    "\n",
    "    # Create a directory to store the csv files\n",
    "    if not os.path.exists(textoutputpath+\"processed\"):\n",
    "            os.mkdir(textoutputpath+\"processed\")\n",
    "\n",
    "    # While the queue is not empty, continue crawling\n",
    "    while queue:\n",
    "\n",
    "        # Get the next URL from the queue\n",
    "        url = queue.pop()\n",
    "        print(url) # for debugging and to see the progress\n",
    "\n",
    "        # Save text from the url to a <url>.txt file\n",
    "        with open(textoutputpath+local_domain+'/'+url[8:].replace(\"/\", \"_\") + \".txt\", \"w\") as f:\n",
    "\n",
    "            # Get the text from the URL using BeautifulSoup\n",
    "            soup = BeautifulSoup(requests.get(url).text, \"html.parser\")\n",
    "\n",
    "            # Get the text but remove the tags\n",
    "            text = soup.get_text()\n",
    "\n",
    "            # If the crawler gets to a page that requires JavaScript, it will stop the crawl\n",
    "            if (\"You need to enable JavaScript to run this app.\" in text):\n",
    "                print(\"Unable to parse page \" + url + \" due to JavaScript being required\")\n",
    "            \n",
    "            # Otherwise, write the text to the file in the text directory\n",
    "            f.write(text)\n",
    "\n",
    "        # Get the hyperlinks from the URL and add them to the queue\n",
    "        for link in get_domain_hyperlinks(local_domain, url, subdomain):\n",
    "            if link not in seen:\n",
    "                queue.append(link)\n",
    "                seen.add(link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/cmpe'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subdomain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.sjsu.edu/cmpe/\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/brett-hartbeck.php\n",
      "https://www.sjsu.edu/cmpe\n",
      "https://www.sjsu.edu/cmpe/faculty\n",
      "https://www.sjsu.edu/cmpe/students/undergraduate-advising/cpt-guide-undergraduates.php\n",
      "https://www.sjsu.edu/cmpe/students/undergraduate-advising\n",
      "https://www.sjsu.edu/cmpe/students/undergraduate-advising/undergraduate-changeofmajor.php#faq\n",
      "https://www.sjsu.edu/cmpe/students/undergraduate-advising/undergraduate-changeofmajor.php#reviewguide\n",
      "https://www.sjsu.edu/cmpe/students/undergraduate-advising/undergraduate-changeofmajor.php#studentguide\n",
      "https://www.sjsu.edu/cmpe/students\n",
      "https://www.sjsu.edu/cmpe/academic/ms-cmpe/msce-faq.php\n",
      "https://www.sjsu.edu/cmpe/academic/ms-cmpe\n",
      "https://www.sjsu.edu/cmpe/academic\n",
      "https://www.sjsu.edu/cmpe/students/graduate-advising/credit-transfer.php\n",
      "https://www.sjsu.edu/cmpe/students/graduate-advising\n",
      "https://www.sjsu.edu/cmpe/faculty/advisors/index.php\n",
      "HTTP Error 404: Not Found\n",
      "https://www.sjsu.edu/cmpe/students/undergraduate-advising/undergraduate-faqs.php\n",
      "https://www.sjsu.edu/cmpe/academic/bscmpe/bcce-technical-electives.php\n",
      "https://www.sjsu.edu/cmpe/academic/bscmpe\n",
      "https://www.sjsu.edu/cmpe/faculty/tenure-line/stas-tiomkin.php\n",
      "https://www.sjsu.edu/cmpe/faculty/tenure-line/haonan-wang.php\n",
      "https://www.sjsu.edu/cmpe/academic/courses.php\n",
      "https://www.sjsu.edu/cmpe/about/news/index.php\n",
      "https://www.sjsu.edu/cmpe/Academic/MSAI/index.php\n",
      "HTTP Error 404: Not Found\n",
      "https://www.sjsu.edu/cmpe/about\n",
      "https://www.sjsu.edu/cmpe/faculty/tenure-line/gokay-saldamli.php\n",
      "https://www.sjsu.edu/cmpe/research/project-expo/2017-fall.php\n",
      "https://www.sjsu.edu/cmpe/research\n",
      "https://www.sjsu.edu/cmpe/academic/bs-software-engineering/bsse-faq.php\n",
      "https://www.sjsu.edu/cmpe/academic/bs-software-engineering\n",
      "https://www.sjsu.edu/cmpe/academic/ms-software-engineering\n",
      "https://www.sjsu.edu/cmpe/research/projects/index.php\n",
      "https://www.sjsu.edu/cmpe/faculty/tenure-line/rod-fatoohi.php\n",
      "https://www.sjsu.edu/cmpe/students/graduate-advising/gpa.php\n",
      "https://www.sjsu.edu/cmpe/resources/hpc.php\n",
      "https://www.sjsu.edu/cmpe/resources\n",
      "<urlopen error [Errno 110] Connection timed out>\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/jahan-ghofraniha.php\n",
      "https://www.sjsu.edu/cmpe/research/index.php\n",
      "https://www.sjsu.edu/cmpe/resources/career-opportunity.php\n",
      "https://www.sjsu.edu/cmpe/students/graduate-advising/cpt.php\n",
      "https://www.sjsu.edu/cmpe/research/research-areas/data-science-analytics.php\n",
      "https://www.sjsu.edu/cmpe/research/research-areas\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/juan-gomez.php\n",
      "https://www.sjsu.edu/cmpe/academic/bscmpe/bsce-probation.php\n",
      "https://www.sjsu.edu/cmpe/students/admission/index.php\n",
      "https://www.sjsu.edu/cmpe/academic/ms-software-engineering/msse-cybersecurity.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/andrew-bond.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/preetpal-kang.php\n",
      "https://www.sjsu.edu/cmpe/academic/bscmpe/index.php\n",
      "https://www.sjsu.edu/cmpe/faculty/officehours.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/gopinath-k-vinodh.php\n",
      "https://www.sjsu.edu/cmpe/research/project-expo/2023-spring.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/index.php\n",
      "https://www.sjsu.edu/cmpe/students/advisors/index.php\n",
      "https://www.sjsu.edu/cmpe/faculty/front-desk-assistant/monica-serna.php\n",
      "HTTP Error 404: Not Found\n",
      "https://www.sjsu.edu/cmpe/students/admission/msse-admission.php\n",
      "https://www.sjsu.edu/cmpe/students/admission\n",
      "https://www.sjsu.edu/cmpe/faculty/tenure-line/weider-yu.php\n",
      "https://www.sjsu.edu/cmpe/students/undergraduate-advising/index.php\n",
      "https://www.sjsu.edu/cmpe/students/graduate-advising/index.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/charles-zhang.php\n",
      "https://www.sjsu.edu/cmpe/research/project-expo/2020-spring.php\n",
      "https://www.sjsu.edu/cmpe/research/research-areas/index.php\n",
      "https://www.sjsu.edu/cmpe/about/computer-engineering.php\n",
      "https://www.sjsu.edu/cmpe/academic/bscmpe/bscmpe-abet.php\n",
      "https://www.sjsu.edu/cmpe/research/thesis.php\n",
      "https://www.sjsu.edu/cmpe/research/project-expo/2021-spring.php\n",
      "https://www.sjsu.edu/cmpe/students/graduate-advising/msse-new-student-welcome.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/hungwen-li.php\n",
      "https://www.sjsu.edu/cmpe/about/announcement/index.php\n",
      "HTTP Error 404: Not Found\n",
      "https://www.sjsu.edu/cmpe/research/research-areas/cloud-computing.php\n",
      "https://www.sjsu.edu/cmpe/academic/ms-cmpe/msce-taking-courses.php\n",
      "https://www.sjsu.edu/cmpe/resources/student-resources.php\n",
      "https://www.sjsu.edu/cmpe/docs/bs-cmpe/cmpe-majorform-fall2014.doc\n",
      "https://www.sjsu.edu/cmpe/docs/bs-cmpe/se-majorform-fall2015.doc\n",
      "https://www.sjsu.edu/cmpe/docs/bs-cmpe/se-programofstudy_fall_2019.doc\n",
      "https://www.sjsu.edu/cmpe/docs/bs-cmpe/advance_registration_form_updated_2017.doc\n",
      "https://www.sjsu.edu/cmpe/docs/bs-cmpe/cmpe-majorform-fall2015-1.doc\n",
      "https://www.sjsu.edu/cmpe/docs/bs-cmpe/cmpe-program_of_study_form_fall_2019.docx\n",
      "https://www.sjsu.edu/cmpe/docs/bs-cmpe/se-majorform-fall2016-19.doc\n",
      "https://www.sjsu.edu/cmpe/docs/bs-cmpe/cmpe-majorform-fall2016-19_1.doc\n",
      "https://www.sjsu.edu/cmpe/academic/cybersecurity-program/cybersecuritycertificatesprogram.php\n",
      "https://www.sjsu.edu/cmpe/academic/cybersecurity-program\n",
      "https://www.sjsu.edu/cmpe/academic/index.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/john-gash.php\n",
      "https://www.sjsu.edu/cmpe/academic/ms-software-engineering/msse-university-learning-goals.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/paul-nguyen.php\n",
      "https://www.sjsu.edu/cmpe/research/project-expo/2023-summer.php\n",
      "https://www.sjsu.edu/cmpe/research/project-expo/Project%20Video\n",
      "HTTP Error 404: Not Found\n",
      "https://www.sjsu.edu/cmpe/students/graduate-advising/prerequisite-classes.php\n",
      "https://www.sjsu.edu/cmpe/faculty/faculty-emeriti/ron-mak.php\n",
      "https://www.sjsu.edu/cmpe/faculty/faculty-emeriti\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/ishie-eswar.php\n",
      "https://www.sjsu.edu/cmpe/faculty/tenure-line/jerry-gao.php\n",
      "https://www.sjsu.edu/cmpe/students/faqs/index.php\n",
      "https://www.sjsu.edu/cmpe/faculty/tenure-line/ahmet-bindal.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/anna-meng.php\n",
      "https://www.sjsu.edu/cmpe/faculty/faculty-emeriti/donald-hung.php\n",
      "https://www.sjsu.edu/cmpe/research/project-expo/2017-spring.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/chao-li-tarng.php\n",
      "https://www.sjsu.edu/cmpe/academic/bs-software-engineering/bsse-probation.php\n",
      "https://www.sjsu.edu/cmpe/resources/ta-position.php\n",
      "https://www.sjsu.edu/cmpe/research/project-expo/2022-spring.php\n",
      "https://www.sjsu.edu/cmpe/research/projects/nsfrings.php\n",
      "https://www.sjsu.edu/cmpe/research/projects\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/shai-silberman.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/michael-larkin.php\n",
      "https://www.sjsu.edu/cmpe/resources/advance-registrationadvising-hold-removal.php\n",
      "https://www.sjsu.edu/cmpe/faculty/lecturer/robert-bruce.php\n",
      "https://www.sjsu.edu/cmpe/research/project-expo/index.php\n"
     ]
    },
    {
     "ename": "ConnectTimeout",
     "evalue": "HTTPSConnectionPool(host='www.sjsu.edu', port=443): Max retries exceeded with url: /cmpe/research/project-expo/index.php (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7f8d5fec79d0>, 'Connection to www.sjsu.edu timed out. (connect timeout=None)'))",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTimeoutError\u001b[0m                              Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/urllib3/connection.py:174\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    173\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 174\u001b[0m     conn \u001b[39m=\u001b[39m connection\u001b[39m.\u001b[39;49mcreate_connection(\n\u001b[1;32m    175\u001b[0m         (\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_dns_host, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mport), \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtimeout, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mextra_kw\n\u001b[1;32m    176\u001b[0m     )\n\u001b[1;32m    178\u001b[0m \u001b[39mexcept\u001b[39;00m SocketTimeout:\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/urllib3/util/connection.py:96\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[39mif\u001b[39;00m err \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m---> 96\u001b[0m     \u001b[39mraise\u001b[39;00m err\n\u001b[1;32m     98\u001b[0m \u001b[39mraise\u001b[39;00m socket\u001b[39m.\u001b[39merror(\u001b[39m\"\u001b[39m\u001b[39mgetaddrinfo returns an empty list\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/urllib3/util/connection.py:86\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     85\u001b[0m     sock\u001b[39m.\u001b[39mbind(source_address)\n\u001b[0;32m---> 86\u001b[0m sock\u001b[39m.\u001b[39;49mconnect(sa)\n\u001b[1;32m     87\u001b[0m \u001b[39mreturn\u001b[39;00m sock\n",
      "\u001b[0;31mTimeoutError\u001b[0m: [Errno 110] Connection timed out",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mConnectTimeoutError\u001b[0m                       Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/urllib3/connectionpool.py:699\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    698\u001b[0m \u001b[39m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 699\u001b[0m httplib_response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_make_request(\n\u001b[1;32m    700\u001b[0m     conn,\n\u001b[1;32m    701\u001b[0m     method,\n\u001b[1;32m    702\u001b[0m     url,\n\u001b[1;32m    703\u001b[0m     timeout\u001b[39m=\u001b[39;49mtimeout_obj,\n\u001b[1;32m    704\u001b[0m     body\u001b[39m=\u001b[39;49mbody,\n\u001b[1;32m    705\u001b[0m     headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[1;32m    706\u001b[0m     chunked\u001b[39m=\u001b[39;49mchunked,\n\u001b[1;32m    707\u001b[0m )\n\u001b[1;32m    709\u001b[0m \u001b[39m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m    710\u001b[0m \u001b[39m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m    711\u001b[0m \u001b[39m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m    712\u001b[0m \u001b[39m# mess.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/urllib3/connectionpool.py:382\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    381\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 382\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_validate_conn(conn)\n\u001b[1;32m    383\u001b[0m \u001b[39mexcept\u001b[39;00m (SocketTimeout, BaseSSLError) \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    384\u001b[0m     \u001b[39m# Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/urllib3/connectionpool.py:1010\u001b[0m, in \u001b[0;36mHTTPSConnectionPool._validate_conn\u001b[0;34m(self, conn)\u001b[0m\n\u001b[1;32m   1009\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mgetattr\u001b[39m(conn, \u001b[39m\"\u001b[39m\u001b[39msock\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mNone\u001b[39;00m):  \u001b[39m# AppEngine might not have  `.sock`\u001b[39;00m\n\u001b[0;32m-> 1010\u001b[0m     conn\u001b[39m.\u001b[39;49mconnect()\n\u001b[1;32m   1012\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m conn\u001b[39m.\u001b[39mis_verified:\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/urllib3/connection.py:358\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    356\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mconnect\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m    357\u001b[0m     \u001b[39m# Add certificate verification\u001b[39;00m\n\u001b[0;32m--> 358\u001b[0m     conn \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_new_conn()\n\u001b[1;32m    359\u001b[0m     hostname \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhost\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/urllib3/connection.py:179\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    178\u001b[0m \u001b[39mexcept\u001b[39;00m SocketTimeout:\n\u001b[0;32m--> 179\u001b[0m     \u001b[39mraise\u001b[39;00m ConnectTimeoutError(\n\u001b[1;32m    180\u001b[0m         \u001b[39mself\u001b[39m,\n\u001b[1;32m    181\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mConnection to \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m timed out. (connect timeout=\u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m)\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    182\u001b[0m         \u001b[39m%\u001b[39m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhost, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtimeout),\n\u001b[1;32m    183\u001b[0m     )\n\u001b[1;32m    185\u001b[0m \u001b[39mexcept\u001b[39;00m SocketError \u001b[39mas\u001b[39;00m e:\n",
      "\u001b[0;31mConnectTimeoutError\u001b[0m: (<urllib3.connection.HTTPSConnection object at 0x7f8d5fec79d0>, 'Connection to www.sjsu.edu timed out. (connect timeout=None)')",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mMaxRetryError\u001b[0m                             Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/requests/adapters.py:439\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    438\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m chunked:\n\u001b[0;32m--> 439\u001b[0m     resp \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39;49murlopen(\n\u001b[1;32m    440\u001b[0m         method\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mmethod,\n\u001b[1;32m    441\u001b[0m         url\u001b[39m=\u001b[39;49murl,\n\u001b[1;32m    442\u001b[0m         body\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mbody,\n\u001b[1;32m    443\u001b[0m         headers\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mheaders,\n\u001b[1;32m    444\u001b[0m         redirect\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    445\u001b[0m         assert_same_host\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    446\u001b[0m         preload_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    447\u001b[0m         decode_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    448\u001b[0m         retries\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_retries,\n\u001b[1;32m    449\u001b[0m         timeout\u001b[39m=\u001b[39;49mtimeout\n\u001b[1;32m    450\u001b[0m     )\n\u001b[1;32m    452\u001b[0m \u001b[39m# Send the request.\u001b[39;00m\n\u001b[1;32m    453\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/urllib3/connectionpool.py:755\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    753\u001b[0m     e \u001b[39m=\u001b[39m ProtocolError(\u001b[39m\"\u001b[39m\u001b[39mConnection aborted.\u001b[39m\u001b[39m\"\u001b[39m, e)\n\u001b[0;32m--> 755\u001b[0m retries \u001b[39m=\u001b[39m retries\u001b[39m.\u001b[39;49mincrement(\n\u001b[1;32m    756\u001b[0m     method, url, error\u001b[39m=\u001b[39;49me, _pool\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m, _stacktrace\u001b[39m=\u001b[39;49msys\u001b[39m.\u001b[39;49mexc_info()[\u001b[39m2\u001b[39;49m]\n\u001b[1;32m    757\u001b[0m )\n\u001b[1;32m    758\u001b[0m retries\u001b[39m.\u001b[39msleep()\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/urllib3/util/retry.py:574\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m    573\u001b[0m \u001b[39mif\u001b[39;00m new_retry\u001b[39m.\u001b[39mis_exhausted():\n\u001b[0;32m--> 574\u001b[0m     \u001b[39mraise\u001b[39;00m MaxRetryError(_pool, url, error \u001b[39mor\u001b[39;00m ResponseError(cause))\n\u001b[1;32m    576\u001b[0m log\u001b[39m.\u001b[39mdebug(\u001b[39m\"\u001b[39m\u001b[39mIncremented Retry for (url=\u001b[39m\u001b[39m'\u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m): \u001b[39m\u001b[39m%r\u001b[39;00m\u001b[39m\"\u001b[39m, url, new_retry)\n",
      "\u001b[0;31mMaxRetryError\u001b[0m: HTTPSConnectionPool(host='www.sjsu.edu', port=443): Max retries exceeded with url: /cmpe/research/project-expo/index.php (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7f8d5fec79d0>, 'Connection to www.sjsu.edu timed out. (connect timeout=None)'))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mConnectTimeout\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[52], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m crawl(full_url, textoutputpath, subdomain)\n",
      "Cell \u001b[0;32mIn[50], line 34\u001b[0m, in \u001b[0;36mcrawl\u001b[0;34m(url, textoutputpath, subdomain)\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[39m# Save text from the url to a <url>.txt file\u001b[39;00m\n\u001b[1;32m     31\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mopen\u001b[39m(textoutputpath\u001b[39m+\u001b[39mlocal_domain\u001b[39m+\u001b[39m\u001b[39m'\u001b[39m\u001b[39m/\u001b[39m\u001b[39m'\u001b[39m\u001b[39m+\u001b[39murl[\u001b[39m8\u001b[39m:]\u001b[39m.\u001b[39mreplace(\u001b[39m\"\u001b[39m\u001b[39m/\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39m_\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m.txt\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mw\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39mas\u001b[39;00m f:\n\u001b[1;32m     32\u001b[0m \n\u001b[1;32m     33\u001b[0m     \u001b[39m# Get the text from the URL using BeautifulSoup\u001b[39;00m\n\u001b[0;32m---> 34\u001b[0m     soup \u001b[39m=\u001b[39m BeautifulSoup(requests\u001b[39m.\u001b[39;49mget(url)\u001b[39m.\u001b[39mtext, \u001b[39m\"\u001b[39m\u001b[39mhtml.parser\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     36\u001b[0m     \u001b[39m# Get the text but remove the tags\u001b[39;00m\n\u001b[1;32m     37\u001b[0m     text \u001b[39m=\u001b[39m soup\u001b[39m.\u001b[39mget_text()\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/requests/api.py:75\u001b[0m, in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     64\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mget\u001b[39m(url, params\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m     65\u001b[0m \u001b[39m    \u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\"\"Sends a GET request.\u001b[39;00m\n\u001b[1;32m     66\u001b[0m \n\u001b[1;32m     67\u001b[0m \u001b[39m    :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     72\u001b[0m \u001b[39m    :rtype: requests.Response\u001b[39;00m\n\u001b[1;32m     73\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 75\u001b[0m     \u001b[39mreturn\u001b[39;00m request(\u001b[39m'\u001b[39;49m\u001b[39mget\u001b[39;49m\u001b[39m'\u001b[39;49m, url, params\u001b[39m=\u001b[39;49mparams, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/requests/api.py:61\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[39m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[1;32m     58\u001b[0m \u001b[39m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[1;32m     59\u001b[0m \u001b[39m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[1;32m     60\u001b[0m \u001b[39mwith\u001b[39;00m sessions\u001b[39m.\u001b[39mSession() \u001b[39mas\u001b[39;00m session:\n\u001b[0;32m---> 61\u001b[0m     \u001b[39mreturn\u001b[39;00m session\u001b[39m.\u001b[39;49mrequest(method\u001b[39m=\u001b[39;49mmethod, url\u001b[39m=\u001b[39;49murl, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/requests/sessions.py:542\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    537\u001b[0m send_kwargs \u001b[39m=\u001b[39m {\n\u001b[1;32m    538\u001b[0m     \u001b[39m'\u001b[39m\u001b[39mtimeout\u001b[39m\u001b[39m'\u001b[39m: timeout,\n\u001b[1;32m    539\u001b[0m     \u001b[39m'\u001b[39m\u001b[39mallow_redirects\u001b[39m\u001b[39m'\u001b[39m: allow_redirects,\n\u001b[1;32m    540\u001b[0m }\n\u001b[1;32m    541\u001b[0m send_kwargs\u001b[39m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 542\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msend(prep, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49msend_kwargs)\n\u001b[1;32m    544\u001b[0m \u001b[39mreturn\u001b[39;00m resp\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/requests/sessions.py:655\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    652\u001b[0m start \u001b[39m=\u001b[39m preferred_clock()\n\u001b[1;32m    654\u001b[0m \u001b[39m# Send the request\u001b[39;00m\n\u001b[0;32m--> 655\u001b[0m r \u001b[39m=\u001b[39m adapter\u001b[39m.\u001b[39;49msend(request, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    657\u001b[0m \u001b[39m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m    658\u001b[0m elapsed \u001b[39m=\u001b[39m preferred_clock() \u001b[39m-\u001b[39m start\n",
      "File \u001b[0;32m~/miniconda3/envs/mypy310/lib/python3.10/site-packages/requests/adapters.py:504\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    501\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(e\u001b[39m.\u001b[39mreason, ConnectTimeoutError):\n\u001b[1;32m    502\u001b[0m     \u001b[39m# TODO: Remove this in 3.0.0: see #2811\u001b[39;00m\n\u001b[1;32m    503\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(e\u001b[39m.\u001b[39mreason, NewConnectionError):\n\u001b[0;32m--> 504\u001b[0m         \u001b[39mraise\u001b[39;00m ConnectTimeout(e, request\u001b[39m=\u001b[39mrequest)\n\u001b[1;32m    506\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(e\u001b[39m.\u001b[39mreason, ResponseError):\n\u001b[1;32m    507\u001b[0m     \u001b[39mraise\u001b[39;00m RetryError(e, request\u001b[39m=\u001b[39mrequest)\n",
      "\u001b[0;31mConnectTimeout\u001b[0m: HTTPSConnectionPool(host='www.sjsu.edu', port=443): Max retries exceeded with url: /cmpe/research/project-expo/index.php (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7f8d5fec79d0>, 'Connection to www.sjsu.edu timed out. (connect timeout=None)'))"
     ]
    }
   ],
   "source": [
    "crawl(full_url, textoutputpath, subdomain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CSV is a common format for storing embeddings. You can use this format with Python by converting the raw text files (which are in the text directory) into Pandas data frames.\n",
    "\n",
    "Converting the text to CSV requires looping through the text files in the text directory created earlier. After opening each file, remove the extra spacing and append the modified text to a list. Then, add the text with the new lines removed to an empty Pandas data frame and write the data frame to a CSV file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_newlines(serie):\n",
    "    serie = serie.str.replace('\\n', ' ')\n",
    "    serie = serie.str.replace('\\\\n', ' ')\n",
    "    serie = serie.str.replace('  ', ' ')\n",
    "    serie = serie.str.replace('  ', ' ')\n",
    "    return serie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./output/text/'"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textoutputpath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sjsu.edu'"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "domain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv\n",
    "def processtxtfiles(inputfolder, outputpath):\n",
    "    # Create a list to store the text files\n",
    "    texts=[]\n",
    "\n",
    "    if not os.path.exists(outputpath):\n",
    "        os.mkdir(outputpath)\n",
    "\n",
    "\n",
    "    # Get all the text files in the text directory\n",
    "    for file in os.listdir(inputfolder + \"/\"):\n",
    "        if file.endswith(\"doc.txt\") or file.endswith(\"docx.txt\"):\n",
    "            continue\n",
    "        # Open the file and read the text\n",
    "        with open(inputfolder + \"/\" + file, \"r\") as f:\n",
    "            text = f.read()\n",
    "\n",
    "            # Omit the first 11 lines and the last 4 lines, then replace -, _, and #update with spaces.\n",
    "            texts.append((file[11:-4].replace('-',' ').replace('_', ' ').replace('#update',''), text))\n",
    "\n",
    "    # Create a dataframe from the list of texts\n",
    "    df = pd.DataFrame(texts, columns = ['fname', 'text'])\n",
    "\n",
    "    # Set the text column to be the raw text with the newlines removed\n",
    "    df['text'] = df.fname + \". \" + remove_newlines(df.text)\n",
    "    df.to_csv(outputpath+'/scraped.csv', escapechar='\\\\')\n",
    "    #csv.writer(f, quoting=csv.QUOTE_NONE, delimiter='|', quotechar='',escapechar='\\\\')    #Or any other appropriate escapechar\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_23999/2931785837.py:3: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  serie = serie.str.replace('\\\\n', ' ')\n"
     ]
    }
   ],
   "source": [
    "inputfolder=\"./output/text/www.sjsu.edu\"\n",
    "outputpath=\"./output/text/processed\"\n",
    "df=processtxtfiles(inputfolder, outputpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fname</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u cmpe faculty faculty emeriti ron mak.php</td>\n",
       "      <td>u cmpe faculty faculty emeriti ron mak.php.   ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u cmpe faculty tenure line ahmet bindal.php</td>\n",
       "      <td>u cmpe faculty tenure line ahmet bindal.php.  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u cmpe research research areas</td>\n",
       "      <td>u cmpe research research areas.     Research A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u cmpe academic cybersecurity program cybersec...</td>\n",
       "      <td>u cmpe academic cybersecurity program cybersec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u cmpe academic bs software engineering bsse f...</td>\n",
       "      <td>u cmpe academic bs software engineering bsse f...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               fname  \\\n",
       "0         u cmpe faculty faculty emeriti ron mak.php   \n",
       "1        u cmpe faculty tenure line ahmet bindal.php   \n",
       "2                     u cmpe research research areas   \n",
       "3  u cmpe academic cybersecurity program cybersec...   \n",
       "4  u cmpe academic bs software engineering bsse f...   \n",
       "\n",
       "                                                text  \n",
       "0  u cmpe faculty faculty emeriti ron mak.php.   ...  \n",
       "1  u cmpe faculty tenure line ahmet bindal.php.  ...  \n",
       "2  u cmpe research research areas.     Research A...  \n",
       "3  u cmpe academic cybersecurity program cybersec...  \n",
       "4  u cmpe academic bs software engineering bsse f...  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'u cmpe academic cybersecurity program cybersecuritycertificatesprogram.php'"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['fname'][3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"u cmpe academic cybersecurity program cybersecuritycertificatesprogram.php.     Cybersecurity Foundations Certificates Program Course Requirements | Computer Engineering   Skip to main content  San José State University  Menu  Visit Campus Tours Maps Parking Silicon Valley Hammer Theatre SJSU Loves SJ Academics Colleges and Departments Majors and Programs Academic Calendar Classes King Library Admissions Tuition and Fees Housing Professional Education Parent and Family Programs Campus Life Health and Wellness Safety SJSU Cares Events Traditions Student Alumni Connection Research and Innovation Office of Research Office of Innovation Research Foundation Student-Faculty Collaborations Become a Research Partner Centers and Institutes About Administration and Leadership Facts and Accomplishments NewsCenter Partnerships Transformation Defined  SJSU Online Athletics Alumni Association Giving to SJSU Canvas one.SJSU  Search    University Menu .Go toSJSU homepage.  Visit Campus Tours Maps Parking Silicon Valley Hammer Theatre SJSU Loves SJ Academics Colleges and Departments Majors and Programs Academic Calendar Classes King Library Admissions Tuition and Fees Housing Professional Education Parents and Family Programs Campus Life Health and Wellness Safety SJSU Cares Events Traditions Student Alumni Connection Research and Innovation Office of Research Office of Innovation Research Foundation Student-Faculty Collaborations Become a Research Partner Centers and Institutes  Athletics SJSU Online About Administration and Leadership Facts and Accomplishments NewsCenter Partnerships Transformation Defined    Home Computer Engineering Academic Programs Cybersecurity Certificates Program Cybersecurity Foundations Certificates Program Course Requirements Certificates Program  Computer EngineeringCollege of Engineering  Computer EngineeringCollege of Engineering Menu About About Computer Engineering News Enrollment Data Why Join Us Academics Academics BS in Computer Engineering BS in Computer Engineering BSCMPE FAQ Probabtion-general Program-Education-Objectives BSCMPE Technical Electives BSCMPE ABET Undergraduate Change of Major Policy BSCMPE Probation BS in Software Engineering BS in Software Engineering BSSE-ABET BSSE Technical Electives BSSE Probation BSSE FAQ MS in Computer Engineering MS in Computer Engineering MSCMPE University Learning Goals MSCMPE Degree Requirements Probation MSCMPE Admission MSCMPE Taking Courses Practical Training MSCMPE Advising MSCMPE FAQ MS in Software Engineering MS in Software Engineering Program Requirements Pre Fall 2020 MSSE Program Requirements MSSE University Learning Goals MSSE Enterprise Software Technologies MSSE Cloud Computing MSSE Software Systems Engineering MSSE Networking Software MSSE Data Science MSSE Cybersecurity MS in Artificial Intelligence MS in Artificial Intelligence MSAI Program Requirements MSAI Admissions Cybersecurity Foundations Certificate Program Cybersecurity Foundations Certificate Program Course Requirements Courses Students Students Undergraduate Advising Undergraduate Advising International Undergraduates CPT Guide Undergraduate Change of Major Undergraduate FAQs Graduate Advising Graduate Advising Advance Registration Admission Prerequisite Classes Change of Major Credit Transfer Curricular Practical Training (CPT) GPA Calculations MS Admission Conditions MSSE New Student Welcome Advisor Directory Admission Admission MSSE Admission Frequently Asked Questions Office Hours Faculty & Staff Faculty & Staff Faculty Directory Faculty Directory Ahmet Bindal Carlos Rojas Gheorghi Guzun Gokay Saldamli Haluk Ozemek Haonan Wang Harry Li Jerry Gao Jorjeta Jetcheva Kaikai Liu Magdalini Eirinaki Mahima Agumbe Suresh Mohamed Fayad Rod Fatoohi Stas Tiomkin Weider Yu Wencen Wu Xiao Su Younghee Park Lecturer Directory Lecturer Directory Ammar Rayes Andrew Bond Anna Meng Bapi Vinnakota Brett Hartbeck Chandrasekar Vuppalapati Chao-Li Tarng Charles Zhang Charley Abboud Dan Harkey Faramarz Mortezaie Frank Butt Frank Lin Gopinath k Vinodh Hungwen Li Ishie Eswar Jahan Ghofraniha John Gash Juan Gomez Kevin Nguyen Kong Li Michael Larkin Paul Nguyen Preetpal Kang Rakesh Ranjan Richard Sinn Robert Bruce Shai Silberman Sithu Aung Front Desk Assistant Faculty Emeriti Faculty Emeriti Donald Hung Ron Mak Sigurd Meldal Office Hours Research Research Research Areas Research Areas AI and ML Data Science & Analytics Embedded & Computer Systems Autonomous & Robotic Systems Networks, Security & IoT Software System & Enterprise Software Cloud Computing & Virtualization Theses Projects Projects NSF RINGS Project Expo Project Expo 2017 Fall Technology Showcase 2017 Spring Technology Showcase 2018 Fall Technology Showcase 2018 Spring Technology Showcase 2018 Summer Technology Showcase 2019 Fall Technology Showcase 2019 Spring Technology Showcase 2020 Spring Technology Showcase 2021 Fall Project Expo 2021 Spring Technology Showcase 2022 Summer Project Expo 2023 Summer Project Expo 2022 Spring Project Expo 2023 Spring Project Expo  Resources Resources TA Positions College of Engineering HPC Student Organizations Graduation For Alumni IAB Members Advance Registration Student Resources Career Opportunities Faculty Resources   About Computer\\xa0Engineering News Enrollment\\xa0Data Why\\xa0Join\\xa0Us Academics BS\\xa0in\\xa0Computer\\xa0Engineering BS\\xa0in\\xa0Software\\xa0Engineering MS\\xa0in\\xa0Computer\\xa0Engineering MS\\xa0in\\xa0Software\\xa0Engineering MS\\xa0in\\xa0Artificial\\xa0Intelligence Cybersecurity\\xa0Foundations\\xa0Certificate\\xa0Program Courses Students Undergraduate\\xa0Advising Graduate\\xa0Advising Advisor\\xa0Directory Admission Frequently\\xa0Asked\\xa0Questions Office\\xa0Hours Faculty & Staff Faculty\\xa0Directory Lecturer\\xa0Directory Front\\xa0Desk\\xa0Assistant Faculty\\xa0Emeriti Office\\xa0Hours Research Research\\xa0Areas Theses Projects Project\\xa0Expo Resources College\\xa0of\\xa0Engineering\\xa0HPC Student\\xa0Organizations Graduation For\\xa0Alumni IAB\\xa0Members Advance\\xa0Registration Student\\xa0Resources Career\\xa0Opportunities Faculty\\xa0Resources   Cybersecurity Foundations Certificates Program Course Requirements 1. You must be a currently enrolled CMPE student at SJSU to pursue this certificate. 2. You need to complete the six core courses before applying for the cybersecurity     certificate. 3. You are only allowed to repeat one of these courses once in order to still be eligible     for the certificate. 4. You will need a minimum 2.0 GPA in all required courses for the certificate (including     1 course repeat). 5. As soon as you receive all grades for the six core course requirements, please     fill out this CMPE Cybersecurity Foundations Certificate Google form to start the application process.\\xa0 This form will be available twice a year: (1) at the end of Spring semester and (2)      at the end of Fall semester. Opens: First day grades are viewable on MySJSU Closes: 5 weekdays after Academic Standing is posted on MySJSUView dates on the appropriate Registration Calendar.\\xa0 6. An application form will be started for you and will be reviewed by the department.     If approved, it will be forwarded to you for signature and then to the Registrar's     office for further processing. 7. Students can take either CMPE 132 (preferred) or CS 166.\\xa0 8. Students can take either CMPE 181 or CMPE 182 to substitute CS/SE 158B starting     Fall 2019. 9. Students can take CMPE 183 to substitute CS/SE 158B starting Fall 2021. Course Requirements\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0\\xa0     Unit CMPE 130. Advanced Algorithm Design \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 3CMPE 132. Information Security \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 3CMPE 138. Database Systems I \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa03CMPE 142. Operating Systems Design \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 3CMPE 148. Computer Networks I \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa03SE 158B. Computer Network Management \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 3 Total Units: \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0 \\xa0     \\xa0 \\xa0 \\xa0\\xa0 \\xa0 \\xa018 Cybersecurity Certificate Coordinator: Dr. Younghee Park If you have any questions, please email undergradadvisor-cmpe-se@sjsu.edu and include     your name and student ID number. If you are interested in obtaining a certificate in cybersecurity, follow the steps     in program requirements.\\xa0 \\xa0  About Computer Engineering News Enrollment Data Why Join Us Academics BS in Computer Engineering BSCMPE FAQ Probabtion-general Program-Education-Objectives BSCMPE Technical Electives BSCMPE ABET Undergraduate Change of Major Policy BSCMPE Probation BS in Software Engineering BSSE-ABET BSSE Technical Electives BSSE Probation BSSE FAQ MS in Computer Engineering MSCMPE University Learning Goals MSCMPE Degree Requirements Probation MSCMPE Admission MSCMPE Taking Courses Practical Training MSCMPE Advising MSCMPE FAQ MS in Software Engineering Program Requirements Pre Fall 2020 MSSE Program Requirements MSSE University Learning Goals MSSE Enterprise Software Technologies MSSE Cloud Computing MSSE Software Systems Engineering MSSE Networking Software MSSE Data Science MSSE Cybersecurity MS in Artificial Intelligence MSAI Program Requirements MSAI Admissions Cybersecurity Foundations Certificate Program Course Requirements Courses Students Undergraduate Advising International Undergraduates CPT Guide Undergraduate Change of Major Undergraduate FAQs Graduate Advising Advance Registration Admission Prerequisite Classes Change of Major Credit Transfer Curricular Practical Training (CPT) GPA Calculations MS Admission Conditions MSSE New Student Welcome Advisor Directory Admission MSSE Admission Frequently Asked Questions Office Hours Faculty & Staff Faculty Directory Ahmet Bindal Carlos Rojas Gheorghi Guzun Gokay Saldamli Haluk Ozemek Haonan Wang Harry Li Jerry Gao Jorjeta Jetcheva Kaikai Liu Magdalini Eirinaki Mahima Agumbe Suresh Mohamed Fayad Rod Fatoohi Stas Tiomkin Weider Yu Wencen Wu Xiao Su Younghee Park Lecturer Directory Ammar Rayes Andrew Bond Anna Meng Bapi Vinnakota Brett Hartbeck Chandrasekar Vuppalapati Chao-Li Tarng Charles Zhang Charley Abboud Dan Harkey Faramarz Mortezaie Frank Butt Frank Lin Gopinath k Vinodh Hungwen Li Ishie Eswar Jahan Ghofraniha John Gash Juan Gomez Kevin Nguyen Kong Li Michael Larkin Paul Nguyen Preetpal Kang Rakesh Ranjan Richard Sinn Robert Bruce Shai Silberman Sithu Aung Front Desk Assistant Faculty Emeriti Donald Hung Ron Mak Sigurd Meldal Office Hours Research Research Areas AI and ML Data Science & Analytics Embedded & Computer Systems Autonomous & Robotic Systems Networks, Security & IoT Software System & Enterprise Software Cloud Computing & Virtualization Theses Projects NSF RINGS Project Expo 2017 Fall Technology Showcase 2017 Spring Technology Showcase 2018 Fall Technology Showcase 2018 Spring Technology Showcase 2018 Summer Technology Showcase 2019 Fall Technology Showcase 2019 Spring Technology Showcase 2020 Spring Technology Showcase 2021 Fall Project Expo 2021 Spring Technology Showcase 2022 Summer Project Expo 2023 Summer Project Expo 2022 Spring Project Expo 2023 Spring Project Expo  Resources College of Engineering HPC Student Organizations Graduation For Alumni IAB Members Advance Registration Student Resources Career Opportunities Faculty Resources   Computer Engineering PLACEHOLDER-FOR-XSL:VALUE-OF(SITE-TITLE) on Facebook PLACEHOLDER-FOR-XSL:VALUE-OF(SITE-TITLE) on LinkedIn  Contact Us Phone: 408-924-4150Fax: 408-924-4153Email: frontdesk-computer-engineering@sjsu.edu Visit Office Engineering Building Room 284One Washington SquareSan Jose, CA 95192-0180 Make a Gift Give to the H&A Fund  Footer San José State University SJSU on Facebook SJSU on Twitter SJSU on LinkedIn SJSU on Instagram SJSU on YouTube  One Washington SquareSan José, CA 95192 408-924-1000 SJSU Online \\xa0  Accessibility Title IX Diversity Land Acknowledgement Privacy  Colleges Business Education Engineering Graduate Studies Health and Human Sciences Humanities and the Arts Professional and Global Education Science Social Science Resources A-Z Index Annual Security Report [pdf] Bookstore Calendars Careers and Jobs Catalog Current Students Emergency Food & Housing Freedom of Speech King Library NewsCenter Parenting Students Parking and Maps Contact Us Contact Form Directory Doing Business with SJSU Emergency Info Report a Web Problem Report a Title IX Complaint        Last Updated Oct 18, 2022 \""
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text'][3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tiktoken\n",
    "\n",
    "# Load the cl100k_base tokenizer which is designed to work with the ada-002 model\n",
    "tokenizer = tiktoken.get_encoding(\"cl100k_base\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fname</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u cmpe faculty faculty emeriti ron mak.php</td>\n",
       "      <td>u cmpe faculty faculty emeriti ron mak.php.   ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u cmpe faculty tenure line ahmet bindal.php</td>\n",
       "      <td>u cmpe faculty tenure line ahmet bindal.php.  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u cmpe research research areas</td>\n",
       "      <td>u cmpe research research areas.     Research A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u cmpe academic cybersecurity program cybersec...</td>\n",
       "      <td>u cmpe academic cybersecurity program cybersec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u cmpe academic bs software engineering bsse f...</td>\n",
       "      <td>u cmpe academic bs software engineering bsse f...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               fname  \\\n",
       "0         u cmpe faculty faculty emeriti ron mak.php   \n",
       "1        u cmpe faculty tenure line ahmet bindal.php   \n",
       "2                     u cmpe research research areas   \n",
       "3  u cmpe academic cybersecurity program cybersec...   \n",
       "4  u cmpe academic bs software engineering bsse f...   \n",
       "\n",
       "                                                text  \n",
       "0  u cmpe faculty faculty emeriti ron mak.php.   ...  \n",
       "1  u cmpe faculty tenure line ahmet bindal.php.  ...  \n",
       "2  u cmpe research research areas.     Research A...  \n",
       "3  u cmpe academic cybersecurity program cybersec...  \n",
       "4  u cmpe academic bs software engineering bsse f...  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df0 = pd.read_csv(outputpath+'/scraped.csv', index_col=0)\n",
    "df0.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u cmpe faculty faculty emeriti ron mak.php</td>\n",
       "      <td>u cmpe faculty faculty emeriti ron mak.php.   ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u cmpe faculty tenure line ahmet bindal.php</td>\n",
       "      <td>u cmpe faculty tenure line ahmet bindal.php.  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u cmpe research research areas</td>\n",
       "      <td>u cmpe research research areas.     Research A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u cmpe academic cybersecurity program cybersec...</td>\n",
       "      <td>u cmpe academic cybersecurity program cybersec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u cmpe academic bs software engineering bsse f...</td>\n",
       "      <td>u cmpe academic bs software engineering bsse f...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0         u cmpe faculty faculty emeriti ron mak.php   \n",
       "1        u cmpe faculty tenure line ahmet bindal.php   \n",
       "2                     u cmpe research research areas   \n",
       "3  u cmpe academic cybersecurity program cybersec...   \n",
       "4  u cmpe academic bs software engineering bsse f...   \n",
       "\n",
       "                                                text  \n",
       "0  u cmpe faculty faculty emeriti ron mak.php.   ...  \n",
       "1  u cmpe faculty tenure line ahmet bindal.php.  ...  \n",
       "2  u cmpe research research areas.     Research A...  \n",
       "3  u cmpe academic cybersecurity program cybersec...  \n",
       "4  u cmpe academic bs software engineering bsse f...  "
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df0.columns = ['title', 'text']\n",
    "df0.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "102"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjHklEQVR4nO3df1CVZf7/8ddB8AgGlJocSDTccK3I1pUitc9CP6A1bWudadswtf1RtmpJzC5q7E7HMjBnvg67Y7lj05gzLWPT5LbubiqnbaMaNFHXzWiz2gjNPLEaAQkdTnJ9/9jxzJ5A48g5F+fA8zHD0H3dF9f95rzPja/uw81xGGOMAAAALIkb6AIAAMDQQvgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYFX8QBfwdd3d3frkk0+UnJwsh8Mx0OUAAIA+MMaovb1dGRkZios7+7WNqAsfn3zyiTIzMwe6DAAAcA6OHDmicePGnXVOyOHj6NGjWr58ubZv367Ozk5NmjRJTz/9tKZNmybpv8ln1apV2rhxo1paWpSXl6cnnnhCl19+eZ/WT05ODhSfkpISanln5Pf7VVNTo6KiIiUkJIRtXUQOPYs99Cz20LPYEs39amtrU2ZmZuDf8bMJKXy0tLRo5syZuu6667R9+3aNHTtW//73v3X++ecH5qxdu1br1q3TM888o0mTJmn16tUqLCzUoUOH+lTQ6ZdaUlJSwh4+kpKSlJKSEnUNQ+/oWeyhZ7GHnsWWWOhXX35lIqTw8fjjjyszM1ObNm0KjF188cWB/zbGqKqqSuXl5Zo7d64kafPmzUpLS1N1dbUWLVoUyuEAAMAgFFL42LZtm2666Sbdfvvtqq2t1UUXXaTFixfrnnvukSQ1NjbK6/WqqKgo8DVOp1P5+fmqq6vrNXz4fD75fL7Adltbm6T/pju/339O31RvTq8VzjURWfQs9tCz2EPPYks09yuUmkIKHx9++KE2bNig0tJSPfTQQ9qzZ48eeOABOZ1OLViwQF6vV5KUlpYW9HVpaWlqamrqdc3KykqtWrWqx3hNTY2SkpJCKa9PPB5P2NdEZNGz2EPPYg89iy3R2K+Ojo4+zw0pfHR3dys3N1cVFRWSpKlTp6qhoUEbNmzQggULAvO+/nqPMeaMrwGtXLlSpaWlge3Tv7BSVFQU9t/58Hg8KiwsjNrXyRCMnsUeehZ76FlsieZ+nX7loi9CCh/p6em67LLLgsYuvfRSvfDCC5Ikl8slSfJ6vUpPTw/MaW5u7nE15DSn0ymn09ljPCEhISIPbKTWReTQs9hDz2IPPYst0divUOoJ6S+czpw5U4cOHQoae++99zRhwgRJUlZWllwuV9DloK6uLtXW1mrGjBmhHAoAAAxSIV35ePDBBzVjxgxVVFToRz/6kfbs2aONGzdq48aNkv77cktJSYkqKiqUnZ2t7OxsVVRUKCkpScXFxRH5BgAAQGwJKXxcddVV+uMf/6iVK1fqkUceUVZWlqqqqjRv3rzAnLKyMnV2dmrx4sWBPzJWU1PTp7/xAQAABr+Q/8LpnDlzNGfOnDPudzgccrvdcrvd/akLAAAMUryrLQAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwKuS7XYC+uHjFX/u9hnOY0dqrpRz3TvlOffNbNPfXR2tmR/wYAACufAAAAMsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsCil8uN1uORyOoA+XyxXYb4yR2+1WRkaGEhMTVVBQoIaGhrAXDQAAYlfIVz4uv/xyHTt2LPBx8ODBwL61a9dq3bp1Wr9+verr6+VyuVRYWKj29vawFg0AAGJXyOEjPj5eLpcr8HHhhRdK+u9Vj6qqKpWXl2vu3LnKycnR5s2b1dHRoerq6rAXDgAAYlN8qF/w/vvvKyMjQ06nU3l5eaqoqNDEiRPV2Ngor9eroqKiwFyn06n8/HzV1dVp0aJFva7n8/nk8/kC221tbZIkv98vv98fanlndHqtcK6JM3MOM/1fI84EfY40nhv9x3kWe+hZbInmfoVSk8MY0+ef7Nu3b1dHR4cmTZqkTz/9VKtXr9a7776rhoYGHTp0SDNnztTRo0eVkZER+Jp7771XTU1N2rlzZ69rut1urVq1qsd4dXW1kpKS+vyNAACAgdPR0aHi4mK1trYqJSXlrHNDCh9fd/LkSX3rW99SWVmZrrnmGs2cOVOffPKJ0tPTA3PuueceHTlyRDt27Oh1jd6ufGRmZur48ePfWHwo/H6/PB6PCgsLlZCQELZ10bscd+9hMxTOOKNHc7v1m71x8nU7wlDV2b3tvinixxjsOM9iDz2LLdHcr7a2No0ZM6ZP4SPkl13+18iRI3XFFVfo/fff12233SZJ8nq9QeGjublZaWlpZ1zD6XTK6XT2GE9ISIjIAxupdRHMdyp8YcHX7QjremfC8yJ8OM9iDz2LLdHYr1Dq6dff+fD5fPrXv/6l9PR0ZWVlyeVyyePxBPZ3dXWptrZWM2bM6M9hAADAIBLSlY9f/vKXuuWWWzR+/Hg1Nzdr9erVamtr08KFC+VwOFRSUqKKigplZ2crOztbFRUVSkpKUnFxcaTqBwAAMSak8PHxxx/rzjvv1PHjx3XhhRfqmmuu0e7duzVhwgRJUllZmTo7O7V48WK1tLQoLy9PNTU1Sk5OjkjxAAAg9oQUPrZs2XLW/Q6HQ263W263uz81AQCAQYz3dgEAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFb1K3xUVlbK4XCopKQkMGaMkdvtVkZGhhITE1VQUKCGhob+1gkAAAaJcw4f9fX12rhxo6ZMmRI0vnbtWq1bt07r169XfX29XC6XCgsL1d7e3u9iAQBA7Dun8PHFF19o3rx5euqpp3TBBRcExo0xqqqqUnl5uebOnaucnBxt3rxZHR0dqq6uDlvRAAAgdp1T+FiyZIlmz56tG2+8MWi8sbFRXq9XRUVFgTGn06n8/HzV1dX1r1IAADAoxIf6BVu2bNH+/ftVX1/fY5/X65UkpaWlBY2npaWpqamp1/V8Pp98Pl9gu62tTZLk9/vl9/tDLe+MTq8VzjVxZs5hpv9rxJmgz5HGc6P/OM9iDz2LLdHcr1BqCil8HDlyRMuWLVNNTY1GjBhxxnkOhyNo2xjTY+y0yspKrVq1qsd4TU2NkpKSQimvTzweT9jXRE9rrw7fWo/mdodvsbN46aWXrBxnKOA8iz30LLZEY786Ojr6PNdhjOnz/1a++OKL+uEPf6hhw4YFxk6dOiWHw6G4uDgdOnRIl1xyifbv36+pU6cG5tx66606//zztXnz5h5r9nblIzMzU8ePH1dKSkqfv5Fv4vf75fF4VFhYqISEhLCti97luHf2ew1nnNGjud36zd44+bp7D6/h9Lb7pogfY7DjPIs99Cy2RHO/2traNGbMGLW2tn7jv98hXfm44YYbdPDgwaCxn/zkJ5o8ebKWL1+uiRMnyuVyyePxBMJHV1eXamtr9fjjj/e6ptPplNPp7DGekJAQkQc2UusimO9U+MKCr9sR1vXOhOdF+HCexR56FluisV+h1BNS+EhOTlZOTk7Q2MiRIzV69OjAeElJiSoqKpSdna3s7GxVVFQoKSlJxcXFoRwKAAAMUiH/wuk3KSsrU2dnpxYvXqyWlhbl5eWppqZGycnJ4T4UAACIQf0OH6+++mrQtsPhkNvtltvt7u/SAABgEOK9XQAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYBXhAwAAWEX4AAAAVhE+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFhF+AAAAFYRPgAAgFWEDwAAYFVI4WPDhg2aMmWKUlJSlJKSounTp2v79u2B/cYYud1uZWRkKDExUQUFBWpoaAh70QAAIHaFFD7GjRunNWvWaO/evdq7d6+uv/563XrrrYGAsXbtWq1bt07r169XfX29XC6XCgsL1d7eHpHiAQBA7AkpfNxyyy26+eabNWnSJE2aNEmPPfaYzjvvPO3evVvGGFVVVam8vFxz585VTk6ONm/erI6ODlVXV0eqfgAAEGPiz/ULT506peeff14nT57U9OnT1djYKK/Xq6KiosAcp9Op/Px81dXVadGiRb2u4/P55PP5AtttbW2SJL/fL7/ff67l9XB6rXCuiTNzDjP9XyPOBH2ONJ4b/cd5FnvoWWyJ5n6FUpPDGBPST/aDBw9q+vTp+vLLL3XeeeepurpaN998s+rq6jRz5kwdPXpUGRkZgfn33nuvmpqatHPnzl7Xc7vdWrVqVY/x6upqJSUlhVIaAAAYIB0dHSouLlZra6tSUlLOOjfkKx/f/va3deDAAX3++ed64YUXtHDhQtXW1gb2OxyOoPnGmB5j/2vlypUqLS0NbLe1tSkzM1NFRUXfWHwo/H6/PB6PCgsLlZCQELZ10bscd+9hMxTOOKNHc7v1m71x8nWf+TkULm+7b4r4MQY7zrPYQ89iSzT36/QrF30RcvgYPny4LrnkEklSbm6u6uvr9dvf/lbLly+XJHm9XqWnpwfmNzc3Ky0t7YzrOZ1OOZ3OHuMJCQkReWAjtS6C+U6FLyz4uh1hXe9MeF6ED+dZ7KFnsSUa+xVKPf3+Ox/GGPl8PmVlZcnlcsnj8QT2dXV1qba2VjNmzOjvYQAAwCAR0pWPhx56SLNmzVJmZqba29u1ZcsWvfrqq9qxY4ccDodKSkpUUVGh7OxsZWdnq6KiQklJSSouLo5U/QAAIMaEFD4+/fRTzZ8/X8eOHVNqaqqmTJmiHTt2qLCwUJJUVlamzs5OLV68WC0tLcrLy1NNTY2Sk5MjUjwAAIg9IYWPp59++qz7HQ6H3G633G53f2oCAACDGO/tAgAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAq0IKH5WVlbrqqquUnJyssWPH6rbbbtOhQ4eC5hhj5Ha7lZGRocTERBUUFKihoSGsRQMAgNgVUviora3VkiVLtHv3bnk8Hn311VcqKirSyZMnA3PWrl2rdevWaf369aqvr5fL5VJhYaHa29vDXjwAAIg98aFM3rFjR9D2pk2bNHbsWO3bt0/f+973ZIxRVVWVysvLNXfuXEnS5s2blZaWpurqai1atCh8lQMAgJgUUvj4utbWVknSqFGjJEmNjY3yer0qKioKzHE6ncrPz1ddXV2v4cPn88nn8wW229raJEl+v19+v78/5QU5vVY418SZOYeZ/q8RZ4I+RxrPjf7jPIs99Cy2RHO/QqnJYYw5p5/sxhjdeuutamlp0euvvy5Jqqur08yZM3X06FFlZGQE5t57771qamrSzp07e6zjdru1atWqHuPV1dVKSko6l9IAAIBlHR0dKi4uVmtrq1JSUs4695yvfCxdulRvvfWW3njjjR77HA5H0LYxpsfYaStXrlRpaWlgu62tTZmZmSoqKvrG4kPh9/vl8XhUWFiohISEsK2L3uW4ewbNUDnjjB7N7dZv9sbJ19378yec3nbfFPFjDHacZ7GHnsWWaO7X6Vcu+uKcwsf999+vbdu26bXXXtO4ceMC4y6XS5Lk9XqVnp4eGG9ublZaWlqvazmdTjmdzh7jCQkJEXlgI7UugvlOhS8s+LodYV3vTHhehA/nWeyhZ7ElGvsVSj0h3e1ijNHSpUu1detWvfLKK8rKygran5WVJZfLJY/HExjr6upSbW2tZsyYEcqhAADAIBXSlY8lS5aourpaf/rTn5ScnCyv1ytJSk1NVWJiohwOh0pKSlRRUaHs7GxlZ2eroqJCSUlJKi4ujsg3AAAAYktI4WPDhg2SpIKCgqDxTZs26e6775YklZWVqbOzU4sXL1ZLS4vy8vJUU1Oj5OTksBQMAABiW0jhoy83xjgcDrndbrnd7nOtCQAADGK8twsAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsCp+oAsAosXFK/460CWE7KM1swe6BAAIGVc+AACAVYQPAABgFeEDAABYRfgAAABWET4AAIBVhA8AAGAV4QMAAFgVcvh47bXXdMsttygjI0MOh0Mvvvhi0H5jjNxutzIyMpSYmKiCggI1NDSEq14AABDjQg4fJ0+e1JVXXqn169f3un/t2rVat26d1q9fr/r6erlcLhUWFqq9vb3fxQIAgNgX8l84nTVrlmbNmtXrPmOMqqqqVF5errlz50qSNm/erLS0NFVXV2vRokX9qxYAAMS8sP7OR2Njo7xer4qKigJjTqdT+fn5qqurC+ehAABAjArre7t4vV5JUlpaWtB4Wlqampqaev0an88nn88X2G5ra5Mk+f1++f3+sNV2eq1wrokzcw4z/V8jzgR9Rk/R9nzmPIs99Cy2RHO/QqkpIm8s53A4graNMT3GTqusrNSqVat6jNfU1CgpKSnstXk8nrCviZ7WXh2+tR7N7Q7fYoPMSy+9NNAl9IrzLPbQs9gSjf3q6Ojo89ywhg+XyyXpv1dA0tPTA+PNzc09roactnLlSpWWlga229ralJmZqaKiIqWkpIStNr/fL4/Ho8LCQiUkJIRtXRty3DsHuoQB4YwzejS3W7/ZGydfd+/hdah7233TQJcQJJbPs6GKnsWWaO7X6Vcu+iKs4SMrK0sul0sej0dTp06VJHV1dam2tlaPP/54r1/jdDrldDp7jCckJETkgY3UupHkOzW0/+H1dTuG/GNwJtH6XI7F82yoo2exJRr7FUo9IYePL774Qh988EFgu7GxUQcOHNCoUaM0fvx4lZSUqKKiQtnZ2crOzlZFRYWSkpJUXFwc6qEAAMAgFHL42Lt3r6677rrA9umXTBYuXKhnnnlGZWVl6uzs1OLFi9XS0qK8vDzV1NQoOTk5fFUDAICYFXL4KCgokDFnvvvA4XDI7XbL7Xb3py4AADBI8d4uAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALAqfqALsC3HvVO+U46BLgMAgCGLKx8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwKohd7cLMJhcvOKvA11CEOcwo7VXn/2uso/WzLZcVXhE22PdF7H6WGPw48oHAACwivABAACsInwAAACrCB8AAMAqwgcAALCKu10AAOgHm3dC9eWOsr4Y6DuhuPIBAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACwivABAACsInwAAACrCB8AAMAqwgcAALCK8AEAAKyKH+gCAAwtF6/460CXgCjG82No4MoHAACwKmLh48knn1RWVpZGjBihadOm6fXXX4/UoQAAQAyJSPh47rnnVFJSovLycv3jH//Q//3f/2nWrFk6fPhwJA4HAABiSETCx7p16/Szn/1MP//5z3XppZeqqqpKmZmZ2rBhQyQOBwAAYkjYf+G0q6tL+/bt04oVK4LGi4qKVFdX12O+z+eTz+cLbLe2tkqSPvvsM/n9/rDV5ff71dHRoXh/nE51O8K2LiInvtuoo6ObnsUQehZdTpw48Y1zTv9sPHHihBISEixUdXbxX50c6BKiWrjOsb48N0LV3t4uSTLGfOPcsIeP48eP69SpU0pLSwsaT0tLk9fr7TG/srJSq1at6jGelZUV7tIQg4oHugCEjJ5FjzH/b6ArQCSE4xyL5HOjvb1dqampZ50TsVttHY7gRGaM6TEmSStXrlRpaWlgu7u7W5999plGjx7d6/xz1dbWpszMTB05ckQpKSlhWxeRQ89iDz2LPfQstkRzv4wxam9vV0ZGxjfODXv4GDNmjIYNG9bjKkdzc3OPqyGS5HQ65XQ6g8bOP//8cJcVkJKSEnUNw9nRs9hDz2IPPYst0dqvb7ricVrYf+F0+PDhmjZtmjweT9C4x+PRjBkzwn04AAAQYyLysktpaanmz5+v3NxcTZ8+XRs3btThw4d13333ReJwAAAghkQkfNxxxx06ceKEHnnkER07dkw5OTl66aWXNGHChEgcrk+cTqcefvjhHi/xIHrRs9hDz2IPPYstg6VfDtOXe2IAAADChPd2AQAAVhE+AACAVYQPAABgFeEDAABYNWTCx5NPPqmsrCyNGDFC06ZN0+uvvz7QJQ16lZWVuuqqq5ScnKyxY8fqtttu06FDh4LmGGPkdruVkZGhxMREFRQUqKGhIWiOz+fT/fffrzFjxmjkyJH6wQ9+oI8//jhoTktLi+bPn6/U1FSlpqZq/vz5+vzzzyP9LQ56lZWVcjgcKikpCYzRs+hz9OhR3XXXXRo9erSSkpL0ne98R/v27Qvsp2fR5auvvtKvf/1rZWVlKTExURMnTtQjjzyi7u7uwJxB3zMzBGzZssUkJCSYp556yrzzzjtm2bJlZuTIkaapqWmgSxvUbrrpJrNp0ybz9ttvmwMHDpjZs2eb8ePHmy+++CIwZ82aNSY5Odm88MIL5uDBg+aOO+4w6enppq2tLTDnvvvuMxdddJHxeDxm//795rrrrjNXXnml+eqrrwJzvv/975ucnBxTV1dn6urqTE5OjpkzZ47V73ew2bNnj7n44ovNlClTzLJlywLj9Cy6fPbZZ2bChAnm7rvvNm+++aZpbGw0L7/8svnggw8Cc+hZdFm9erUZPXq0+ctf/mIaGxvN888/b8477zxTVVUVmDPYezYkwsfVV19t7rvvvqCxyZMnmxUrVgxQRUNTc3OzkWRqa2uNMcZ0d3cbl8tl1qxZE5jz5ZdfmtTUVPP73//eGGPM559/bhISEsyWLVsCc44ePWri4uLMjh07jDHGvPPOO0aS2b17d2DOrl27jCTz7rvv2vjWBp329naTnZ1tPB6Pyc/PD4QPehZ9li9fbq699toz7qdn0Wf27Nnmpz/9adDY3LlzzV133WWMGRo9G/Qvu3R1dWnfvn0qKioKGi8qKlJdXd0AVTU0tba2SpJGjRolSWpsbJTX6w3qjdPpVH5+fqA3+/btk9/vD5qTkZGhnJycwJxdu3YpNTVVeXl5gTnXXHONUlNT6fE5WrJkiWbPnq0bb7wxaJyeRZ9t27YpNzdXt99+u8aOHaupU6fqqaeeCuynZ9Hn2muv1d/+9je99957kqR//vOfeuONN3TzzTdLGho9i9i72kaL48eP69SpUz3e1C4tLa3Hm98hcowxKi0t1bXXXqucnBxJCjz+vfWmqakpMGf48OG64IILesw5/fVer1djx47tccyxY8fS43OwZcsW7d+/X/X19T320bPo8+GHH2rDhg0qLS3VQw89pD179uiBBx6Q0+nUggUL6FkUWr58uVpbWzV58mQNGzZMp06d0mOPPaY777xT0tA4zwZ9+DjN4XAEbRtjeowhcpYuXaq33npLb7zxRo9959Kbr8/pbT49Dt2RI0e0bNky1dTUaMSIEWecR8+iR3d3t3Jzc1VRUSFJmjp1qhoaGrRhwwYtWLAgMI+eRY/nnntOzz77rKqrq3X55ZfrwIEDKikpUUZGhhYuXBiYN5h7NuhfdhkzZoyGDRvWI+U1Nzf3SJWIjPvvv1/btm3T3//+d40bNy4w7nK5JOmsvXG5XOrq6lJLS8tZ53z66ac9jvuf//yHHodo3759am5u1rRp0xQfH6/4+HjV1tbqd7/7neLj4wOPJz2LHunp6brsssuCxi699FIdPnxYEudZNPrVr36lFStW6Mc//rGuuOIKzZ8/Xw8++KAqKyslDY2eDfrwMXz4cE2bNk0ejydo3OPxaMaMGQNU1dBgjNHSpUu1detWvfLKK8rKygran5WVJZfLFdSbrq4u1dbWBnozbdo0JSQkBM05duyY3n777cCc6dOnq7W1VXv27AnMefPNN9Xa2kqPQ3TDDTfo4MGDOnDgQOAjNzdX8+bN04EDBzRx4kR6FmVmzpzZ4xb29957L/BGnpxn0aejo0NxccH//A4bNixwq+2Q6NkA/JKrdadvtX366afNO++8Y0pKSszIkSPNRx99NNClDWq/+MUvTGpqqnn11VfNsWPHAh8dHR2BOWvWrDGpqalm69at5uDBg+bOO+/s9XaycePGmZdfftns37/fXH/99b3eTjZlyhSza9cus2vXLnPFFVdExe1kg8H/3u1iDD2LNnv27DHx8fHmscceM++//775wx/+YJKSksyzzz4bmEPPosvChQvNRRddFLjVduvWrWbMmDGmrKwsMGew92xIhA9jjHniiSfMhAkTzPDhw813v/vdwO2eiBxJvX5s2rQpMKe7u9s8/PDDxuVyGafTab73ve+ZgwcPBq3T2dlpli5dakaNGmUSExPNnDlzzOHDh4PmnDhxwsybN88kJyeb5ORkM2/ePNPS0mLhuxz8vh4+6Fn0+fOf/2xycnKM0+k0kydPNhs3bgzaT8+iS1tbm1m2bJkZP368GTFihJk4caIpLy83Pp8vMGew98xhjDEDeeUFAAAMLYP+dz4AAEB0IXwAAACrCB8AAMAqwgcAALCK8AEAAKwifAAAAKsIHwAAwCrCBwAAsIrwAQAArCJ8AAAAqwgfAADAKsIHAACw6v8Dgqz7kbNvDQcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Tokenize the text and save the number of tokens to a new column\n",
    "df0['n_tokens'] = df0.text.apply(lambda x: len(tokenizer.encode(x)))\n",
    "\n",
    "# Visualize the distribution of the number of tokens per row using a histogram\n",
    "df0.n_tokens.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>n_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u cmpe faculty faculty emeriti ron mak.php</td>\n",
       "      <td>u cmpe faculty faculty emeriti ron mak.php.   ...</td>\n",
       "      <td>2003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u cmpe faculty tenure line ahmet bindal.php</td>\n",
       "      <td>u cmpe faculty tenure line ahmet bindal.php.  ...</td>\n",
       "      <td>2027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u cmpe research research areas</td>\n",
       "      <td>u cmpe research research areas.     Research A...</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u cmpe academic cybersecurity program cybersec...</td>\n",
       "      <td>u cmpe academic cybersecurity program cybersec...</td>\n",
       "      <td>2520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u cmpe academic bs software engineering bsse f...</td>\n",
       "      <td>u cmpe academic bs software engineering bsse f...</td>\n",
       "      <td>2698</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0         u cmpe faculty faculty emeriti ron mak.php   \n",
       "1        u cmpe faculty tenure line ahmet bindal.php   \n",
       "2                     u cmpe research research areas   \n",
       "3  u cmpe academic cybersecurity program cybersec...   \n",
       "4  u cmpe academic bs software engineering bsse f...   \n",
       "\n",
       "                                                text  n_tokens  \n",
       "0  u cmpe faculty faculty emeriti ron mak.php.   ...      2003  \n",
       "1  u cmpe faculty tenure line ahmet bindal.php.  ...      2027  \n",
       "2  u cmpe research research areas.     Research A...      2017  \n",
       "3  u cmpe academic cybersecurity program cybersec...      2520  \n",
       "4  u cmpe academic bs software engineering bsse f...      2698  "
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df0.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "102"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "split the longer lines into smaller chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_tokens = 500\n",
    "\n",
    "# Function to split the text into chunks of a maximum number of tokens\n",
    "def split_into_many(text, max_tokens = max_tokens):\n",
    "\n",
    "    # Split the text into sentences\n",
    "    sentences = text.split('. ')\n",
    "\n",
    "    # Get the number of tokens for each sentence\n",
    "    n_tokens = [len(tokenizer.encode(\" \" + sentence)) for sentence in sentences]\n",
    "\n",
    "    chunks = []\n",
    "    tokens_so_far = 0\n",
    "    chunk = []\n",
    "\n",
    "    # Loop through the sentences and tokens joined together in a tuple\n",
    "    for sentence, token in zip(sentences, n_tokens):\n",
    "\n",
    "        # If the number of tokens so far plus the number of tokens in the current sentence is greater\n",
    "        # than the max number of tokens, then add the chunk to the list of chunks and reset\n",
    "        # the chunk and tokens so far\n",
    "        if tokens_so_far + token > max_tokens:\n",
    "            chunks.append(\". \".join(chunk) + \".\")\n",
    "            chunk = []\n",
    "            tokens_so_far = 0\n",
    "\n",
    "        # If the number of tokens in the current sentence is greater than the max number of\n",
    "        # tokens, go to the next sentence\n",
    "        if token > max_tokens:\n",
    "            continue\n",
    "\n",
    "        # Otherwise, add the sentence to the chunk and add the number of tokens to the total\n",
    "        chunk.append(sentence)\n",
    "        tokens_so_far += token + 1\n",
    "\n",
    "    return chunks\n",
    "\n",
    "\n",
    "shortened = []\n",
    "\n",
    "# Loop through the dataframe\n",
    "for row in df0.iterrows():\n",
    "\n",
    "    # If the text is None, go to the next row\n",
    "    if row[1]['text'] is None:\n",
    "        continue\n",
    "\n",
    "    # If the number of tokens is greater than the max number of tokens, split the text into chunks\n",
    "    if row[1]['n_tokens'] > max_tokens:\n",
    "        shortened += split_into_many(row[1]['text'])\n",
    "\n",
    "    # Otherwise, add the text to the list of shortened texts\n",
    "    else:\n",
    "        shortened.append( row[1]['text'] )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualizing the updated histogram again can help to confirm if the rows were successfully split into shortened sections."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgvUlEQVR4nO3de3BU5f3H8c8mLEtCE+RSsqxEiTXWS5BaUEqwJhazDgNeytRbbKXVViyXIY0tgrRlsZpIOpOmhYrFOkjrZPAPxDLjhayjBpnUEblUiA7amkZE0owak2BwsybP7w+a7W/dqEHO7j5H368ZZtyzJw9Pvuys7zmbzXqMMUYAAAAWyUj3BgAAAD6OQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgnWHp3sDn0d/fr7fffls5OTnyeDzp3g4AABgCY4y6u7sVCASUkfHp10hcGShvv/228vPz070NAADwORw6dEgTJ0781HNcGSg5OTmSjn+Dubm5jq0bjUbV0NCgYDAor9fr2LqIx5xTgzmnDrNODeacGsmcc1dXl/Lz82P/H/80rgyUgZd1cnNzHQ+U7Oxs5ebm8uBPIuacGsw5dZh1ajDn1EjFnIfy4xn8kCwAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKwzLN0bANxk0vLH072FIfNlGtVclO5dAMDnwxUUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGCdEw6UHTt26IorrlAgEJDH49Fjjz0Wd78xRqFQSIFAQFlZWSotLVVzc3PcOZFIREuWLNG4ceM0cuRIXXnllXrrrbdO6hsBAABfHCccKB988IGmTJmidevWDXp/TU2NamtrtW7dOu3atUt+v19lZWXq7u6OnVNRUaGtW7dq8+bN2rlzp44ePaq5c+eqr6/v838nAADgC+OEfw/K7NmzNXv27EHvM8aorq5OK1eu1Lx58yRJmzZtUl5enurr67VgwQJ1dnbqwQcf1F//+ldddtllkqSHH35Y+fn5evrpp3X55ZefxLcDAAC+CBz9GZSWlha1tbUpGAzGjvl8PpWUlKipqUmStHv3bkWj0bhzAoGAioqKYucAAIAvN0d/k2xbW5skKS8vL+54Xl6eWltbY+cMHz5co0ePTjhn4Os/LhKJKBKJxG53dXVJkqLRqKLRqGP7H1jLyTWRyM1z9mWadG9hyHwZx/fqxjm7jZsf027CnFMjmXM+kTWT8qvuPR5P3G1jTMKxj/u0c6qrq7V69eqE4w0NDcrOzv78G/0E4XDY8TWRyI1zduOvjnfjnN2KWacGc06NZMy5p6dnyOc6Gih+v1/S8askEyZMiB1vb2+PXVXx+/3q7e1VR0dH3FWU9vZ2FRcXD7ruihUrVFlZGbvd1dWl/Px8BYNB5ebmOrb/aDSqcDissrIyeb1ex9ZFPDfPuSi0Pd1bGDJfhtFvpvW7cs5u4+bHtJsw59RI5pwHXgEZCkcDpaCgQH6/X+FwWBdccIEkqbe3V42NjVqzZo0kaerUqfJ6vQqHw7r22mslSUeOHNGBAwdUU1Mz6Lo+n08+ny/huNfrTcqDNFnrIp4b5xzp+/QrgTZy45zdilmnBnNOjWTM+UTWO+FAOXr0qP75z3/Gbre0tGjfvn0aM2aMTjvtNFVUVKiqqkqFhYUqLCxUVVWVsrOzVV5eLkkaNWqUbrnlFt1+++0aO3asxowZo5///OeaPHly7F09AADgy+2EA+Wll17SpZdeGrs98NLL/Pnz9dBDD2nZsmU6duyYFi5cqI6ODk2fPl0NDQ3KycmJfc3vfvc7DRs2TNdee62OHTumWbNm6aGHHlJmZqYD3xIAAHC7Ew6U0tJSGfPJ72TweDwKhUIKhUKfeM6IESO0du1arV279kT/egAA8CXAZ/EAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOo4HykcffaRf/vKXKigoUFZWls444wzddddd6u/vj51jjFEoFFIgEFBWVpZKS0vV3Nzs9FYAAIBLOR4oa9as0f33369169bp1VdfVU1NjX77299q7dq1sXNqampUW1urdevWadeuXfL7/SorK1N3d7fT2wEAAC7keKD8/e9/11VXXaU5c+Zo0qRJ+t73vqdgMKiXXnpJ0vGrJ3V1dVq5cqXmzZunoqIibdq0ST09Paqvr3d6OwAAwIWGOb3gxRdfrPvvv1+vvfaazjrrLP3jH//Qzp07VVdXJ0lqaWlRW1ubgsFg7Gt8Pp9KSkrU1NSkBQsWJKwZiUQUiURit7u6uiRJ0WhU0WjUsb0PrOXkmkjk5jn7Mk26tzBkvozje3XjnN3GzY9pN2HOqZHMOZ/Imo4Hyh133KHOzk6dffbZyszMVF9fn+655x7dcMMNkqS2tjZJUl5eXtzX5eXlqbW1ddA1q6urtXr16oTjDQ0Nys7Odvg7kMLhsONrIpEb51xzUbp3cOLcOGe3YtapwZxTIxlz7unpGfK5jgfKI488oocfflj19fU677zztG/fPlVUVCgQCGj+/Pmx8zweT9zXGWMSjg1YsWKFKisrY7e7urqUn5+vYDCo3Nxcx/YejUYVDodVVlYmr9fr2LqI5+Y5F4W2p3sLQ+bLMPrNtH5Xztlt3PyYdhPmnBrJnPPAKyBD4Xig/OIXv9Dy5ct1/fXXS5ImT56s1tZWVVdXa/78+fL7/ZKOX0mZMGFC7Ova29sTrqoM8Pl88vl8Cce9Xm9SHqTJWhfx3DjnSN/gEW0zN87ZrZh1ajDn1EjGnE9kPcd/SLanp0cZGfHLZmZmxt5mXFBQIL/fH3fpqLe3V42NjSouLnZ6OwAAwIUcv4JyxRVX6J577tFpp52m8847T3v37lVtba1uvvlmScdf2qmoqFBVVZUKCwtVWFioqqoqZWdnq7y83OntAAAAF3I8UNauXatf/epXWrhwodrb2xUIBLRgwQL9+te/jp2zbNkyHTt2TAsXLlRHR4emT5+uhoYG5eTkOL0dAADgQo4HSk5Ojurq6mJvKx6Mx+NRKBRSKBRy+q8HAABfAHwWDwAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALBOUgLl8OHD+v73v6+xY8cqOztb3/jGN7R79+7Y/cYYhUIhBQIBZWVlqbS0VM3NzcnYCgAAcKFhTi/Y0dGhmTNn6tJLL9WTTz6p8ePH61//+pdOOeWU2Dk1NTWqra3VQw89pLPOOkt33323ysrKdPDgQeXk5Di9JQAA0mrS8sfTvYUh82Ua1VyU7l0kIVDWrFmj/Px8bdy4MXZs0qRJsf82xqiurk4rV67UvHnzJEmbNm1SXl6e6uvrtWDBAqe3BAAAXMbxQNm2bZsuv/xyXXPNNWpsbNSpp56qhQsX6ic/+YkkqaWlRW1tbQoGg7Gv8fl8KikpUVNT06CBEolEFIlEYre7urokSdFoVNFo1LG9D6zl5JpI5OY5+zJNurcwZL6M43t145zdxs2PaTdx85x57tAJr+kxxjg6tREjRkiSKisrdc011+jFF19URUWF/vSnP+mmm25SU1OTZs6cqcOHDysQCMS+7tZbb1Vra6u2b9+esGYoFNLq1asTjtfX1ys7O9vJ7QMAgCTp6elReXm5Ojs7lZub+6nnOn4Fpb+/X9OmTVNVVZUk6YILLlBzc7PWr1+vm266KXaex+OJ+zpjTMKxAStWrFBlZWXsdldXl/Lz8xUMBj/zGzwR0WhU4XBYZWVl8nq9jq2LeG6ec1EoMaBt5csw+s20flfO2W3c/Jh2EzfPmeeO4wZeARkKxwNlwoQJOvfcc+OOnXPOOdqyZYskye/3S5La2to0YcKE2Dnt7e3Ky8sbdE2fzyefz5dw3Ov1JuVBmqx1Ec+Nc470DR7RNnPjnN2KWaeGG+fMc8f/1hwqx99mPHPmTB08eDDu2GuvvabTTz9dklRQUCC/369wOBy7v7e3V42NjSouLnZ6OwAAwIUcv4Lys5/9TMXFxaqqqtK1116rF198URs2bNCGDRskHX9pp6KiQlVVVSosLFRhYaGqqqqUnZ2t8vJyp7cDAABcyPFAufDCC7V161atWLFCd911lwoKClRXV6cbb7wxds6yZct07NgxLVy4UB0dHZo+fboaGhr4HSgAAEBSEgJFkubOnau5c+d+4v0ej0ehUEihUCgZfz0AAHA5PosHAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWCfpgVJdXS2Px6OKiorYMWOMQqGQAoGAsrKyVFpaqubm5mRvBQAAuERSA2XXrl3asGGDzj///LjjNTU1qq2t1bp167Rr1y75/X6VlZWpu7s7mdsBAAAukbRAOXr0qG688UY98MADGj16dOy4MUZ1dXVauXKl5s2bp6KiIm3atEk9PT2qr69P1nYAAICLJC1QFi1apDlz5uiyyy6LO97S0qK2tjYFg8HYMZ/Pp5KSEjU1NSVrOwAAwEWGJWPRzZs3a8+ePdq1a1fCfW1tbZKkvLy8uON5eXlqbW0ddL1IJKJIJBK73dXVJUmKRqOKRqNObTu2lpNrIpGb5+zLNOnewpD5Mo7v1Y1zdhs3P6bdxM1z5rlDJ7ym44Fy6NAhLV26VA0NDRoxYsQnnufxeOJuG2MSjg2orq7W6tWrE443NDQoOzv75DY8iHA47PiaSOTGOddclO4dnDg3ztmtmHVquHHOPHcc19PTM+RzPcYYR7Puscce03e/+11lZmbGjvX19cnj8SgjI0MHDx7UmWeeqT179uiCCy6InXPVVVfplFNO0aZNmxLWHOwKSn5+vt555x3l5uY6tvdoNKpwOKyysjJ5vV7H1kU8N8+5KLQ93VsYMl+G0W+m9btyzm7j5se0m7h5zjx3HNfV1aVx48aps7PzM///7fgVlFmzZmn//v1xx370ox/p7LPP1h133KEzzjhDfr9f4XA4Fii9vb1qbGzUmjVrBl3T5/PJ5/MlHPd6vUl5kCZrXcRz45wjfYNf5bOZG+fsVsw6Ndw4Z547/rfmUDkeKDk5OSoqKoo7NnLkSI0dOzZ2vKKiQlVVVSosLFRhYaGqqqqUnZ2t8vJyp7cDAABcKCk/JPtZli1bpmPHjmnhwoXq6OjQ9OnT1dDQoJycnHRsBwAAWCYlgfLcc8/F3fZ4PAqFQgqFQqn46wEAgMvwWTwAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwjuOBUl1drQsvvFA5OTkaP368rr76ah08eDDuHGOMQqGQAoGAsrKyVFpaqubmZqe3AgAAXMrxQGlsbNSiRYv0wgsvKBwO66OPPlIwGNQHH3wQO6empka1tbVat26ddu3aJb/fr7KyMnV3dzu9HQAA4ELDnF7wqaeeiru9ceNGjR8/Xrt379Yll1wiY4zq6uq0cuVKzZs3T5K0adMm5eXlqb6+XgsWLHB6SwAAwGUcD5SP6+zslCSNGTNGktTS0qK2tjYFg8HYOT6fTyUlJWpqaho0UCKRiCKRSOx2V1eXJCkajSoajTq214G1nFwTidw8Z1+mSfcWhsyXcXyvbpyz27j5Me0mbp4zzx064TU9xpikTc0Yo6uuukodHR16/vnnJUlNTU2aOXOmDh8+rEAgEDv31ltvVWtrq7Zv356wTigU0urVqxOO19fXKzs7O1nbBwAADurp6VF5ebk6OzuVm5v7qecm9QrK4sWL9fLLL2vnzp0J93k8nrjbxpiEYwNWrFihysrK2O2uri7l5+crGAx+5jd4IqLRqMLhsMrKyuT1eh1bF/HcPOeiUGJA28qXYfSbaf2unLPbuPkx7SZunjPPHccNvAIyFEkLlCVLlmjbtm3asWOHJk6cGDvu9/slSW1tbZowYULseHt7u/Ly8gZdy+fzyefzJRz3er1JeZAma13Ec+OcI32DR7TN3Dhnt2LWqeHGOfPc8b81h8rxd/EYY7R48WI9+uijeuaZZ1RQUBB3f0FBgfx+v8LhcOxYb2+vGhsbVVxc7PR2AACACzl+BWXRokWqr6/X3/72N+Xk5KitrU2SNGrUKGVlZcnj8aiiokJVVVUqLCxUYWGhqqqqlJ2drfLycqe3AwAAXMjxQFm/fr0kqbS0NO74xo0b9cMf/lCStGzZMh07dkwLFy5UR0eHpk+froaGBuXk5Di9HQAA4EKOB8pQ3hTk8XgUCoUUCoWc/usBAMAXAJ/FAwAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsMyzdG7BRUWi7In2edG/jhPz73jnp3gIAAI7hCgoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArMPbjAEAruLGXwWBE8cVFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdfpMsAHyJTVr+eLq3MGS+TKOai9K9C6QKV1AAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB3eZgx8wRWFtivS50n3Nr7QBt7+yqwB53AFBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADW4W3GXxBu/ERS3pIJAPgkXEEBAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWCetgXLfffepoKBAI0aM0NSpU/X888+nczsAAMASaQuURx55RBUVFVq5cqX27t2rb3/725o9e7befPPNdG0JAABYIm2BUltbq1tuuUU//vGPdc4556iurk75+flav359urYEAAAskZbP4unt7dXu3bu1fPnyuOPBYFBNTU0J50ciEUUikdjtzs5OSdJ7772naDTq2L6i0ah6eno0LJqhvn4+IyZZhvUb9fT0M+ckY86pw6xTgzmnxsCc3333XXm9XkfX7u7uliQZYz57H47+zUP0zjvvqK+vT3l5eXHH8/Ly1NbWlnB+dXW1Vq9enXC8oKAgaXtEcpWnewNfEsw5dZh1ajDn1Ej2nLu7uzVq1KhPPSetn2bs8cQXsDEm4ZgkrVixQpWVlbHb/f39eu+99zR27NhBz/+8urq6lJ+fr0OHDik3N9exdRGPOacGc04dZp0azDk1kjlnY4y6u7sVCAQ+89y0BMq4ceOUmZmZcLWkvb094aqKJPl8Pvl8vrhjp5xyStL2l5uby4M/BZhzajDn1GHWqcGcUyNZc/6sKycD0vJDssOHD9fUqVMVDofjjofDYRUXF6djSwAAwCJpe4mnsrJSP/jBDzRt2jTNmDFDGzZs0JtvvqnbbrstXVsCAACWSFugXHfddXr33Xd111136ciRIyoqKtITTzyh008/PV1bks/n06pVqxJeToKzmHNqMOfUYdapwZxTw5Y5e8xQ3usDAACQQnwWDwAAsA6BAgAArEOgAAAA6xAoAADAOgTKf913330qKCjQiBEjNHXqVD3//PPp3pKr7NixQ1dccYUCgYA8Ho8ee+yxuPuNMQqFQgoEAsrKylJpaamam5vjzolEIlqyZInGjRunkSNH6sorr9Rbb72Vwu/CftXV1brwwguVk5Oj8ePH6+qrr9bBgwfjzmHWJ2/9+vU6//zzY7+oasaMGXryySdj9zPj5KiurpbH41FFRUXsGLN2RigUksfjifvj9/tj91s5ZwOzefNm4/V6zQMPPGBeeeUVs3TpUjNy5EjT2tqa7q25xhNPPGFWrlxptmzZYiSZrVu3xt1/7733mpycHLNlyxazf/9+c91115kJEyaYrq6u2Dm33XabOfXUU004HDZ79uwxl156qZkyZYr56KOPUvzd2Ovyyy83GzduNAcOHDD79u0zc+bMMaeddpo5evRo7BxmffK2bdtmHn/8cXPw4EFz8OBBc+eddxqv12sOHDhgjGHGyfDiiy+aSZMmmfPPP98sXbo0dpxZO2PVqlXmvPPOM0eOHIn9aW9vj91v45wJFGPMRRddZG677ba4Y2effbZZvnx5mnbkbh8PlP7+fuP3+829994bO/bhhx+aUaNGmfvvv98YY8z7779vvF6v2bx5c+ycw4cPm4yMDPPUU0+lbO9u097ebiSZxsZGYwyzTqbRo0ebP//5z8w4Cbq7u01hYaEJh8OmpKQkFijM2jmrVq0yU6ZMGfQ+W+f8pX+Jp7e3V7t371YwGIw7HgwG1dTUlKZdfbG0tLSora0tbsY+n08lJSWxGe/evVvRaDTunEAgoKKiIv4dPkVnZ6ckacyYMZKYdTL09fVp8+bN+uCDDzRjxgxmnASLFi3SnDlzdNlll8UdZ9bOev311xUIBFRQUKDrr79eb7zxhiR755zWTzO2wTvvvKO+vr6EDynMy8tL+DBDfD4Dcxxsxq2trbFzhg8frtGjRyecw7/D4Iwxqqys1MUXX6yioiJJzNpJ+/fv14wZM/Thhx/qK1/5irZu3apzzz039mTMjJ2xefNm7dmzR7t27Uq4j8ezc6ZPn66//OUvOuuss/Sf//xHd999t4qLi9Xc3GztnL/0gTLA4/HE3TbGJBzDyfk8M+bf4ZMtXrxYL7/8snbu3JlwH7M+eV//+te1b98+vf/++9qyZYvmz5+vxsbG2P3M+OQdOnRIS5cuVUNDg0aMGPGJ5zHrkzd79uzYf0+ePFkzZszQ1772NW3atEnf+ta3JNk35y/9Szzjxo1TZmZmQgG2t7cn1CQ+n4GfFP+0Gfv9fvX29qqjo+MTz8H/LFmyRNu2bdOzzz6riRMnxo4za+cMHz5cZ555pqZNm6bq6mpNmTJFv//975mxg3bv3q329nZNnTpVw4YN07Bhw9TY2Kg//OEPGjZsWGxWzNp5I0eO1OTJk/X6669b+5j+0gfK8OHDNXXqVIXD4bjj4XBYxcXFadrVF0tBQYH8fn/cjHt7e9XY2Bib8dSpU+X1euPOOXLkiA4cOMC/w/9jjNHixYv16KOP6plnnlFBQUHc/cw6eYwxikQizNhBs2bN0v79+7Vv377Yn2nTpunGG2/Uvn37dMYZZzDrJIlEInr11Vc1YcIEex/TSfnRW5cZeJvxgw8+aF555RVTUVFhRo4caf7973+ne2uu0d3dbfbu3Wv27t1rJJna2lqzd+/e2Fu17733XjNq1Cjz6KOPmv3795sbbrhh0LewTZw40Tz99NNmz5495jvf+Q5vFfyYn/70p2bUqFHmueeei3u7YE9PT+wcZn3yVqxYYXbs2GFaWlrMyy+/bO68806TkZFhGhoajDHMOJn+/7t4jGHWTrn99tvNc889Z9544w3zwgsvmLlz55qcnJzY/+dsnDOB8l9//OMfzemnn26GDx9uvvnNb8betomhefbZZ42khD/z5883xhx/G9uqVauM3+83Pp/PXHLJJWb//v1xaxw7dswsXrzYjBkzxmRlZZm5c+eaN998Mw3fjb0Gm7Eks3Hjxtg5zPrk3XzzzbHng69+9atm1qxZsTgxhhkn08cDhVk7Y+D3mni9XhMIBMy8efNMc3Nz7H4b5+wxxpjkXJsBAAD4fL70P4MCAADsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwzv8Brr232B4ZqgAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.DataFrame(shortened, columns = ['text'])\n",
    "df['n_tokens'] = df.text.apply(lambda x: len(tokenizer.encode(x)))\n",
    "df.n_tokens.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>n_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u cmpe faculty faculty emeriti ron mak.php.   ...</td>\n",
       "      <td>160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u cmpe faculty tenure line ahmet bindal.php.  ...</td>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u cmpe research research areas.     Research A...</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u cmpe academic cybersecurity program cybersec...</td>\n",
       "      <td>165</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  n_tokens\n",
       "0  u cmpe faculty faculty emeriti ron mak.php.   ...       160\n",
       "1  u cmpe faculty tenure line ahmet bindal.php.  ...       161\n",
       "2                                                  .         1\n",
       "3  u cmpe research research areas.     Research A...       154\n",
       "4  u cmpe academic cybersecurity program cybersec...       165"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "222"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59379"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df['n_tokens'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The content is now broken down into smaller chunks and a simple request can be sent to the OpenAI API specifying the use of the new text-embedding-ada-002 model to create the embeddings:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./output/text/processed'"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputpath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>n_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>CMPE 195A (2 units) has to be taken concurre...</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>u cmpe research project expo 2020 spring.php. ...</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  n_tokens\n",
       "8    CMPE 195A (2 units) has to be taken concurre...       171\n",
       "9  u cmpe research project expo 2020 spring.php. ...       163"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[8:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(outputpath+'/split.csv', escapechar='\\\\')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mypy310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}