Edureka_NLP_Assignment_1_Part_A.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Edureka NLP Assignment 1 - Part A.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO+I/c7+T7+0C5MHBf1XK6P",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/nandinib1999/edureka-assignments/blob/main/Edureka_NLP_Assignment_1_Part_A.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2VZeHDy-uOob",
        "outputId": "49fd0e75-9f2b-4d0f-cca9-fcb0f9178ee8"
      },
      "source": [
        "import nltk\n",
        "from nltk.tokenize import word_tokenize, sent_tokenize\n",
        "from collections import Counter\n",
        "nltk.download('punkt')\n",
        "\n",
        "def Tokenize(text):\n",
        "  sents = sent_tokenize(text)\n",
        "  all_words = []\n",
        "  for sent in sents:\n",
        "    words = word_tokenize(sent)\n",
        "    words = [x.lower() for x in words]\n",
        "    all_words.extend(words)\n",
        "  \n",
        "  word_freq_count = Counter(all_words)\n",
        "  print(\"word_freq_count:\", word_freq_count)\n",
        "  least_common = word_freq_count.most_common()[-5:]\n",
        "  print(\"5 least occuring tokens:\", least_common)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zPoC-96ktjBT",
        "outputId": "2ae32d1c-2150-4c0a-d556-bdd4a2175802"
      },
      "source": [
        "user_input = input()\n",
        "Tokenize(user_input)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout. The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English. Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose\n",
            "word_freq_count: Counter({',': 6, 'a': 5, 'it': 3, 'the': 3, 'of': 3, '.': 3, 'ipsum': 3, 'is': 2, 'that': 2, 'will': 2, 'by': 2, 'readable': 2, 'content': 2, 'page': 2, 'using': 2, 'lorem': 2, 'as': 2, 'here': 2, \"'\": 2, 'many': 2, 'and': 2, 'web': 2, 'their': 2, 'sometimes': 2, 'long': 1, 'established': 1, 'fact': 1, 'reader': 1, 'be': 1, 'distracted': 1, 'when': 1, 'looking': 1, 'at': 1, 'its': 1, 'layout': 1, 'point': 1, 'has': 1, 'more-or-less': 1, 'normal': 1, 'distribution': 1, 'letters': 1, 'opposed': 1, 'to': 1, \"'content\": 1, 'making': 1, 'look': 1, 'like': 1, 'english': 1, 'desktop': 1, 'publishing': 1, 'packages': 1, 'editors': 1, 'now': 1, 'use': 1, 'default': 1, 'model': 1, 'text': 1, 'search': 1, 'for': 1, \"'lorem\": 1, 'uncover': 1, 'sites': 1, 'still': 1, 'in': 1, 'infancy': 1, 'various': 1, 'versions': 1, 'have': 1, 'evolved': 1, 'over': 1, 'years': 1, 'accident': 1, 'on': 1, 'purpose': 1})\n",
            "5 least occuring tokens: [('over', 1), ('years', 1), ('accident', 1), ('on', 1), ('purpose', 1)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-8PPFSlLvAo4",
        "outputId": "61fcf04b-ca42-44a1-eb1d-ba3c82de97bf"
      },
      "source": [
        "from nltk.corpus import stopwords\n",
        "import matplotlib.pyplot as plt\n",
        "nltk.download('stopwords')\n",
        "stop_words = stopwords.words('english')\n",
        "\n",
        "def RemoveStopWords(text):\n",
        "  filtered_sentence = [w.lower() for w in text.split(\" \") if not w.lower() in stop_words]\n",
        "  stop_words_in_sent = [w.lower() for w in text.split(\" \") if w.lower() in stop_words]\n",
        "  word_freq_count = Counter(stop_words_in_sent)\n",
        "  print(\"stopword frequency count\")\n",
        "  for stop, cnt in word_freq_count.items():\n",
        "    print(stop, '->', cnt)\n",
        "  plt.figure(figsize=(15, 5))\n",
        "  plt.bar(word_freq_count.keys(), word_freq_count.values())\n",
        "  plt.xticks(rotation=90)\n",
        "  plt.title(\"Stopword count\")\n",
        "  plt.show()"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 868
        },
        "id": "Yblgz5SZxzK8",
        "outputId": "b941b363-0579-4f64-e63c-a9f4a47e2881"
      },
      "source": [
        "user_input = input()\n",
        "RemoveStopWords(user_input)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are many variations of passages of Lorem Ipsum available, but the majority have suffered alteration in some form, by injected humour, or randomised words which don't look even slightly believable. If you are going to use a passage of Lorem Ipsum, you need to be sure there isn't anything embarrassing hidden in the middle of text. All the Lorem Ipsum generators on the Internet tend to repeat predefined chunks as necessary, making this the first true generator on the Internet. It uses a dictionary of over 200 Latin words, combined with a handful of model sentence structures, to generate Lorem Ipsum which looks reasonable. The generated Lorem Ipsum is therefore always free from repetition, injected humour, or non-characteristic words etc.\n",
            "stopword frequency count\n",
            "there -> 2\n",
            "are -> 2\n",
            "of -> 6\n",
            "but -> 1\n",
            "the -> 7\n",
            "have -> 1\n",
            "in -> 2\n",
            "some -> 1\n",
            "by -> 1\n",
            "or -> 2\n",
            "which -> 2\n",
            "don't -> 1\n",
            "if -> 1\n",
            "you -> 2\n",
            "to -> 4\n",
            "a -> 3\n",
            "be -> 1\n",
            "isn't -> 1\n",
            "all -> 1\n",
            "on -> 2\n",
            "as -> 1\n",
            "this -> 1\n",
            "it -> 1\n",
            "over -> 1\n",
            "with -> 1\n",
            "is -> 1\n",
            "from -> 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2AAAAFSCAYAAACUv7wrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3debhlZ1kn7N9DEuYwdUoQQhGgMUAjYzE1YW4EhEYbEEUDgmK1dAvYoHZo7Rb8FEO3ojQq3ZFRBiEgCDIoIFOAkEACMtMMhkkRGRMjEBKf74+1DjkpqlIncNa7q3bd93Wdq85ee9d63r332cNvrXeo7g4AAADLu9SqGwAAAHCoEMAAAAAGEcAAAAAGEcAAAAAGEcAAAAAGEcAAAAAGEcAAOKRU1XOq6jdX3Q4ADk0CGAD7VFXHVdU7quprVfXlqnp7Vd16vu5hVfW2VbfxYOdxBDi0HL7qBgBwYKqqKyV5VZJHJjk5yaWT3DHJN1fZrkuiqg7r7gtW3Q4A2OAMGAD78gNJ0t1/2t0XdPfXu/t13f2+qrpRkv+T5PZV9U9V9dUkqaorV9WfVNU/VtWnqurXqupS83UPm8+g/cF8Ru0jVXX3+bq7VtX7NwpX1eur6l2bLp9SVT86/36jqnpzVX21qj5YVffbdLvnVNXTq+o1VXVukrtW1S2q6syqOqeqXpzkshd3p6vq56rqw/PtP1RVt9xC3TdX1SM2Xb7IWa2q6qr6+ar62Pz//7Ame30cAVhfAhgA+/L/klxQVc+tqntX1VU3rujuDyf5+SSndvcVu/sq81VPS3LlJNdLcuckD03y8E37vG2STyQ5KsmvJ3lZVV0tyTuT3KCqjqqqI5LcNMk1q+rIqrpckl1JTpmv+4skr0vyfUkeleQFVXXspho/meS3khyZ5PQkf57keUmuluQlSR6wrztcVT+W5Alzu6+U5H5JvrTFuvtz3yS3nu/bg5Lc82IeRwDWlAAGwF5199lJjkvSSf44yT9W1Sur6up7u31VHZbkJ5I8vrvP6e6zkvxukodsutkXkvx+d3+ru1+c5KNJ7tPdX0/yriR3SnKrJH+T5O1J7pDkdkk+1t1fmn+/YpITu/u87n5jpm6SD95U4xXd/fbu/pckN09yxKaaL53r7MsjkvzP7n5XTz7e3Z/aYt39ObG7v9rdn07yprltABxijAEDYJ/mMzQPS5KqumGS5yf5/ew9eByVKex8atO2TyW51qbLn+vu3uP6a86/vyXJXZJ8dv79K5nOon1zvpz5tp+Zw9W+anxm0+/X3EfNfbl2pjN0e9pK3f35/Kbf/zlToAPgEOMMGABb0t0fSfKcJDfZ2LTHTb6Y5FtJrrNp284kn9t0+VpVVXtc/3fz7xsB7E7z72/JFMDunAsD2N8lufbGuLJ91Njcrr/fR819+UyS6+9l+/7qnpvk8puuu8bF1NjTno8jAGtMAANgr6rqhlX1uKo6er587Uxnvt453+QfkhxdVZdOknm2wZOT/NY8dus6SR6b6azZhu9L8uiqOmIeb3WjJK+Zr3tHkmOT3CbJ6d39wUxh7rZJ3jrf5rRMZ49+Zd7HXZL8+yQv2sfdODXJ+Ztq3n/e/748I8kvVdWt5kky/vV8P/ZX971J7l9Vl6+qf53kZy+mxp4u8jgCsN4EMAD25ZxM4ee0eUbBdyb5QJLHzde/MckHk3y+qr44b3tUprNBn0zytiQvTPKsTfs8LckNMp0t+60kD5zHdqW7z01yZpIPdvd58+1PTfKp7v7CfJvzMgWfe8/7+KMkD53Pzn2H+fb3z9SN8stJfjzJy/Z1h7v7JXO7Xjjf/z9PcrUt1P29JOdlClPPTfKCfdXYi709jgCsqbpot3gAWEZVPSzJI7r7uFW3BQBWxRkwAACAQQQwAACAQXRBBAAAGMQZMAAAgEEEMAAAgEEOX2KnRx11VB9zzDFL7BoAAOCAd8YZZ3yxu3fsuX2RAHbMMcfk3e9+9xK7BgAAOOBV1af2tl0XRAAAgEEEMAAAgEEEMAAAgEEEMAAAgEEEMAAAgEEEMAAAgEEEMAAAgEH2G8Cq6tiqeu+mn7Or6hdHNA4AAGCd7Hch5u7+aJKbJ0lVHZbkc0levnC7AAAA1s4l7YJ49ySf6O69ruoMAADAvl3SAPYTSf50iYYAAACsu/12QdxQVZdOcr8kj9/H9buT7E6SnTt3bkvjYE/HnPDqxfZ91on3WWzfAACQXLIzYPdOcmZ3/8Peruzuk7p7V3fv2rFjx/a0DgAAYI1ckgD24Oh+CAAA8F3bUgCrqiskuUeSly3bHAAAgPW1pTFg3X1ukn+1cFsAAADW2iWdBREAAIDvkgAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwiAAGAAAwyJYCWFVdpapeWlUfqaoPV9Xtl24YAADAujl8i7d7apK/7O4HVtWlk1x+wTYBAACspf0GsKq6cpI7JXlYknT3eUnOW7ZZAAAA62crXRCvm+Qfkzy7qt5TVc+oqiss3C4AAIC1s5UuiIcnuWWSR3X3aVX11CQnJPnvm29UVbuT7E6SnTt3bnc72YJjTnj1Yvs+68T7LLZvAAA4VGzlDNhnk3y2u0+bL780UyC7iO4+qbt3dfeuHTt2bGcbAQAA1sJ+A1h3fz7JZ6rq2HnT3ZN8aNFWAQAArKGtzoL4qCQvmGdA/GSShy/XJAAAgPW0pQDW3e9NsmvhtgAAAKy1LS3EDAAAwPdOAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABhEAAMAABjk8K3cqKrOSnJOkguSnN/du5ZsFAAAwDraUgCb3bW7v7hYSwAAANacLogAAACDbDWAdZLXVdUZVbV7yQYBAACsq612QTyuuz9XVd+X5PVV9ZHufuvmG8zBbHeS7Ny5c5ubCQAAcPDb0hmw7v7c/O8Xkrw8yW32cpuTuntXd+/asWPH9rYSAABgDew3gFXVFarqyI3fk/xQkg8s3TAAAIB1s5UuiFdP8vKq2rj9C7v7LxdtFQAAwBrabwDr7k8mudmAtgAAAKw109ADAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMIoABAAAMsuUAVlWHVdV7qupVSzYIAABgXV2SM2CPSfLhpRoCAACw7rYUwKrq6CT3SfKMZZsDAACwvrZ6Buz3k/xKkn9ZsC0AAABr7fD93aCq7pvkC919RlXd5WJutzvJ7iTZuXPntjUQAFbhmBNevdi+zzrxPovtG4AD21bOgN0hyf2q6qwkL0pyt6p6/p436u6TuntXd+/asWPHNjcTAADg4LffANbdj+/uo7v7mCQ/keSN3X384i0DAABYM9YBAwAAGGS/Y8A26+43J3nzIi0BAABYc86AAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADCKAAQAADLLfAFZVl62q06vqb6rqg1X1xBENAwAAWDeHb+E230xyt+7+p6o6Isnbquq13f3OhdsGAACwVvYbwLq7k/zTfPGI+aeXbBQAAMA62tIYsKo6rKrem+QLSV7f3act2ywAAID1s5UuiOnuC5LcvKqukuTlVXWT7v7A5ttU1e4ku5Nk586d295QAFh3x5zw6kX2e9aJ91lkvwBccpdoFsTu/mqSNyW5116uO6m7d3X3rh07dmxX+wAAANbGVmZB3DGf+UpVXS7JPZJ8ZOmGAQAArJutdEH8/iTPrarDMgW2k7v7Vcs2CwAAYP1sZRbE9yW5xYC2AAAArLVLNAYMAACA754ABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMIgABgAAMMh+A1hVXbuq3lRVH6qqD1bVY0Y0DAAAYN0cvoXbnJ/kcd19ZlUdmeSMqnp9d39o4bYBAACslf2eAevuv+/uM+ffz0ny4STXWrphAAAA6+YSjQGrqmOS3CLJaUs0BgAAYJ1tpQtikqSqrpjkz5L8YnefvZfrdyfZnSQ7d+7ctgZup2NOePUi+z3rxPsMrXdxNTn4+bvZfqt4TEe/3xwKPKYcDLyHA/uzpTNgVXVEpvD1gu5+2d5u090ndfeu7t61Y8eO7WwjAADAWtjKLIiV5JlJPtzdT1m+SQAAAOtpK2fA7pDkIUnuVlXvnX9+eOF2AQAArJ39jgHr7rclqQFtAQAAWGuXaBZEAAAAvnsCGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCACGAAAwCD7DWBV9ayq+kJVfWBEgwAAANbVVs6APSfJvRZuBwAAwNrbbwDr7rcm+fKAtgAAAKw1Y8AAAAAGOXy7dlRVu5PsTpKdO3du1245wB1zwqsX2/dZJ95nsX1fEkvdx3W/f8m+7+O6P6arMPp5PBRe+4eCQ+HvZt3fb9bpMd1XzXW6jx7TsTUPVNt2Bqy7T+ruXd29a8eOHdu1WwAAgLWhCyIAAMAgW5mG/k+TnJrk2Kr6bFX97PLNAgAAWD/7HQPW3Q8e0RAAAIB1pwsiAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIAIYAADAIFsKYFV1r6r6aFV9vKpOWLpRAAAA62i/AayqDkvyh0nuneTGSR5cVTdeumEAAADrZitnwG6T5OPd/cnuPi/Ji5L8yLLNAgAAWD9bCWDXSvKZTZc/O28DAADgEqjuvvgbVD0wyb26+xHz5YckuW13/8Iet9udZPd88dgkH93+5g51VJIvrnG9VdR0Hw/+equo6T4e/PVWUXPd662ipvt48NdbRU338eCvt4qaq7iPS7hOd+/Yc+PhW/iPn0ty7U2Xj563XUR3n5TkpO+6eQeYqnp3d+9a13qrqOk+Hvz1VlHTfTz4662i5rrXW0VN9/Hgr7eKmu7jwV9vFTVXcR9H2koXxHcluUFVXbeqLp3kJ5K8ctlmAQAArJ/9ngHr7vOr6heS/FWSw5I8q7s/uHjLAAAA1sxWuiCmu1+T5DULt+VAM7o75Sq6b7qP6h0MNd3Hg7/eKmque71V1HQfD/56q6jpPh789VZRc22GNe3NfifhAAAAYHtsZQwYAAAA20AAAwAAGEQAY1FVdYetbINVqcm193/Lg9vI12JVPXn+98eW2P+hrKquWlW3qao7bfysuk3braous5VtsAqHymcGyzIGbFZVV0/ypCTX7O57V9WNk9y+u5+5YM1K8lNJrtfdv1FVO5Nco7tP3+Y6z+vuh1TVY7r7qdu57y3UPrO7b7m/bdtYbxXP4w8keXqSq3f3Tarqpknu192/uWDNf5vkmGyaSKe7/2TBescluUF3P7uqdiS5Ynf/7YL1XpbkmUle293/slSdTfXe390/uHSdPWqOfg6HvRar6v1JbprkjKVe63up+dC9bV/yMZ3rXj3JreeLp3f3Fxas9Ygkj8m0Hud7k9wuyandfbcFa67iPXXo58Zoc5h8QL7z9f8bC9V7TJJnJzknyTOS3CLJCd39unWot7nunt9xlvres4rPjLnu6M+NOyR5b3efW1XHJ7llkqd296cWqnfTfOf9e9kStVbNGbALPSfTVPvXnC//vyS/uHDNP0py+yQPni+fk+QPF6hzq6q6ZpKfmY+eXm3zzwL1UlW3r6rHJdlRVY/d9POETMsZLOU5Gf88/nGSxyf5VpJ09/syrZe3iKp6XpLfSXJcpi9+t06y2GKFVfXrSf5rpvuYJEckef5S9WZ/lOQnk3ysqk6sqmMXrndmVd16/zfbHiOfwxW9Fv8yyVeS3LSqzt70c05Vnb1QzVtv+rljkickud9CtZIkVfWgJKcn+bEkD0pyWlU9cMGSj8l0/z7V3XfN9MX2qwvWSwa+p1bVNarqVkkuV1W3qKpbzj93SXL5Beqds8ff54i/0yR5RZIfSXJ+knM3/SzlZ7r77CQ/lOSqSR6S5MQ1qrfhp/ey7WEL1Rr6mZGM/+yfPT3JP1fVzZI8LsknkiwS+KrqWUmelengxL+ff+67RK0DwZamoT9EHNXdJ1fV45Nvr392wcI1b9vdt6yq98w1vzIvdr3d/k+Sv05yvSRnbNpeSXrevt0uneSKmf7Gjty0/ewkS35BWcXzePnuPn06oflt5y9Yb1eSG/e409f/IdMXvTOTpLv/rqqOvPj/8r3p7jckeUNVXTnTAYo3VNVnMoXd53f3t7a55G2THF9VZ2X6IlRTM/qm21xnw8jncPhrsbt/OckvV9UruvtHlqixl5qP2ny5qq6S5EULl/3VJLfeOOs1nx1+Q5KXLlTvG939japKVV2muz8y4ODEyPfUe2b6wnx0kt/N9DpMpr/V/7bdxbp70fexi3F0d99rYL2Nx/GHkzyvuz9Ye3xgHcz1qurBmQ7YXbeqXrnpqiOTfHmhsrdN8lNV9amM+cxIxn/2J8n53d1V9SNJ/qC7n1lVP7tQrdt1940X2vcBRwC70LlV9a8yBZJU1e2SfG3hmt+qqsM21dyRZNu7W3X3/07yv6vq6ZnC2MaYgbd2999sd7255luSvKWqnrPUqep9WMXz+MWquv6mmg9M8vcL1vtAkmssXGOz8+Y34I37d4URRefn8SFJjk/yniQvyHTk76eT3GWby90z05HaO86X35plzywMew5X+FrMqPC1D+cmue7CNS61R5fDL2XZniWfnYPlnyd5fVV9JcnSz+mw99Tufm6S51bVA7r7z5aosdn+eoB091Jf3t9RVT/Y3e9faP97OqOqXpfp9fD4+QDakl27R9d7R6b30qMyBfcN5yR530I177nQfi/O6M/+JDlnPvhyfJI7VdWlMvWCWcKpVXXj7v7QQvs/oBgDNquqWyZ5WpKbZPoj35HkgXN3sqVq/lSSH8/Up/a5mY5G/1p3v2Sheo9J8ogkL8t0tOZHk/xxdz9tiXpzzTdl/uDebKkxCyt6Hq+XacHAf5up29XfJvmpBftIvynJzTN1ffrmxvbuXqS7VVX9UpIbJLlHkt9O8jNJXrjw383Lkxyb5HlJnt3dn9903bu7e1u7XYx6bVTVX2R6PRyZQc9hVT17rvm17v4v273/fdR8W3cfV1XnzLVr87/dfaUFam48tsnUtfJGSU7u7hO2u9ammv8zyc2S/Om86ceTvK+7/+tSNTfVvnOSKyf5y+4+b8E6G++p/ybJB7Pge2pdOI7v60t9Du5R729z4d/lhs1/p0v0DklVfSjTe+onM73+Fz17Mn9pvnmmL86XyRRUrrXUe/imep/s7q/OAf5aS34Or0INGhu9is+NTbWvkens4ru6+5Sa5iq4yxLjzub3tFcm+XwGvC5WTQBLMp+FenSmD5ljMz3pH12gm9PmmpfKNID6y0nuPtf86+7+8II135dp8PS58+UrZBrAvdgf99yff8NlM/XtPb+7f2XBmodn0PM41zusuy+YH89Ldfc5C9e78962z2c6lqp5j0z9+SvJX3X365eqNde7d6YvfHfIdOT0bUme3t3fWKjekNfGvp67DUs8h5tqntfdp273/g8Uezy252caJ/XZhWs+Oclpmc7MJskpmbrRLB7ARqmqyyb5hUxH/M9JcmqSpy3xWqxpvGmSnNPdT9nu/e+n9tUyhaLLbmxb6j21qq6TvZxxX/Cg3ZDJW6rqhnO32L1OltLdZ25nvU119zzg8+2rstwBn1/P1CXw2O7+gZrG2b+ku7d9ZtlVfG6sQlV9PMljk7w/m86Yju65MYoANquq07v7NoNrvqe7bzGw3vszjVf4xnz5spmOaoye/W3Rx7rGzxL06UyTDrw4yRsH988epqqulIs+pkt1z0lVnZxp3McL5k0/meQq3b3ItOajXxtVdd0kf7+p3uUyzaJ51hL1DhU1cEbCud7eZut73zodsR39WlyFfQSUd3T33ReqN7Q3ysb7W5J3dvfNq+qGSZ7U3fff5jondffuuZfGnnqpni+rUFXvzTw2euN73NKv/ap68p4Hd/a2bZtqrSLUntrdt9/u/R6ojAG70Nur6g8yfYn+9mxESx2xmf11VT0gycsGfWl/dqZZul4+X/7RTFN9L2aPPvaXynTE6MoL1ntekutn+hDdGCjeWWjWntkNM83U85+TPLOqXpXkRd39tu0ssoo3xLnuf0zyxCTfyHRUasnJWzbcZI/BuG+au+0sZfRr4yWZuqxuuGDettisWjVNJ/yEJNfJ9N6/aDer0WqakfB/JXlzpvv2tKr65e7e9gkxquqRSf5TkuvNZ083HJnk7dtdb8VGvxY3xkP/XL7zQNrPLFRyY3bJd3b3XTcCykK1kuRnM50p3Tjj/uTMZxYXqjdk8pbu3j3/e9ft3vcBaBVjo++RaUbize69l23fs+4+bv535EQ176mqFyb5i1y0i+VaTkMvgF3o5vO/m9fh6CRLHrH5j5lOt55fVd/Iwl+ku/spVfXmXNhd5uHd/Z4lam1yRi4MC+cnOSvTh89Shs8S1N3/nOTkJCdX1VWTPDXJW7LNU3yv6A0xSX4p05ewLw6seWZV3a6735kkVXXbJO9eqtgKXhuHbx63093n1TIzoG72zCT/JdNrcumZQVdh5IyEL0zy2kxjIjePMTtnyTPDKzL0tTh7RabunG/ImL/V0bNLVi56vy7IRcehbbfhk7eM7omyAidX1f9NcpWq+rlMY6P/eIlCh9ABn8tlCl4/tGlbZzpTvHYEsNkqjth095F763e+cM0zM08nPsiNM71xHJfphXRKlv3wXsUsQRt9tH88yb0y3b8Hjay/sE8k+ecRheauMp1psPg75u6dnemszUeWrD34tfGPVXW/7n5lktQ0xe/SAfdr3f3ahWus0rAZCbv7a5lmAnzw/m57sFrlazHT0h4jx9GNDihDz7h393+Yf33C3D3wypm6zS9iRT1Rhuru35nHRp+dacz5/1hwbPQhccCnux++6jaMZAzYbB478KQk1+zue1fVjTMNyl/sTXF0v/NVGDV+oFY7S9BZmaZJPznJKze6layLqrpF5i8Muehj+ugFal3n4q5fl8G4NS1b8IJMi9tWks8keWh3f3zBmidmOiv7slz0eRx5QGYxtcIZCdfRKl+LVfWbmT4LX7NUjYupPXJ2yW9P3jKgN8owVfXhjF+vaqiqemySF3f35wbUulJ3n137WDZhXUJYVR2dqRvuxkQmpyR5TC88mdKqCGCzqnptpi+Zv9rdN6tpJr33LDlBxaiBsatUVR/aY/zAXrdtQ507Z/oi++Qkm2dYrCRP7u7bbme9PWpfqbvPXmr/q1ZVp2eahXDPmYmeu7JGrYmqumKSdPc/Dai11gPjq+rRmYLsxsxyp3T3yy/mv3CAmse5XiHTgYJvZeHu+WyvqnpJkkd399CeKCPVNAvigzLNZP3iTDMg/sNCtV7V3fetfSybsEbjeF+f6Wzf8+ZNx2da0uceq2vVcnRBvNBR3X1yTQvOpbvPr6ql+56P7ne+CkPGD/Q8DWtVHdF7TMk6zzC3pPOq6j9nmjZ98xTGSw0YH+2I7n7sqhuxDqrq+O5+/nz0dPP2JNNYtKVqHwID478v03IiZyZ5VpK/Wm1z+G6tYJwr22CPnigfmg/eDemJMlp3PzHJE6vqppnOtr+lqj7b3f9ugVr3nX99e6bx5ad099LdgFdhR3c/e9Pl51TVL66sNQtbpH/8QercmhYL3JjR5naZ+vgvac9+56/IwgNjR6mq98+DRW+VafzAWfPRm1MzTZSx3fUeOZ9RPLaq3rfp52+TLL344/MyjTu7Z6Y3x6MzrZezLl5bVbur6vur6mobP6tu1EFqY6asI/fxs5iqunJVPaWq3j3//G5VLTYj6Wjd/WuZxtM+M8nDknysqp40d/fkIFJVd9iYVa6qjp//bneuul3s1+8keUqSy2ca1/akJL87b7v6Ctu1pC9kWjj4S5kOAi3pmUm+P9MMr5+sqpfWtKTBuvjS/Ho/bP45PtPjupZ0QZzN/bGfluQmmSZy2JHkgYb7ipEAAAV+SURBVD1o5fZR/c5HGT1+YP4iedWsYJBqzeu51bwGSFUdkekI1e2WrDvKHGL3tDbdHg4VVfVnmd7bNrqOPiTJzdapy3OSVNXNkjw804Q4b8o0tvb1veDi72yv+eDdzZLcNMlzkjwjyYO6+2IXpOXAUIfG+nj/KVMXxB2ZlhA5ubsXXZ5hrntYpqErd03y80m+3t03XLruCPP3xqcluX2mkyHvyNSV9dMrbdhCdEGcdfeZcwg6NlP/2o9297cG1l+Llcw3jJ4sYcWzkm38nXy1qm6S6WjY0kfChunu6666Deumxq9zlCTX7+4HbLr8xJoWE10L85Hgh2aaTfIZSX65u79VVZdK8rFcdGwoB7bzu7trmh30D7r7mVW15PIlbIM6dKZLT5JrZ5pE7U65cLbQRVXVX2fqRXFqpgkqvr3sxsFuDpZPWqduqvsjgF3UbXLhF6JbVtW6rVvBMk6qaf2vX0vyyiRXTPLfV9uk7TOf0Xtkpg+aZFro9v+OPECxhkavc5QkX6+q43peILymhZm/Pqj2CFdLcv89D/50979U1X338X84MJ0zj8c+Psmd5hC9+BdcvmeHxHTps88neX6mWWUryfOr6qTuXmox7WQaTnGrTD21vpbpoO+p3X3Qv4939wVVdZ2quvQ69ALbCl0QZ7WPdSuWmGqb9VJVl0nygEzhfeNLQnf3b+zzPx1EquoZme7X5q5rF3T3I1bXqoNbVb23u2++/1tua82bZVqHZ2Pc11eS/PSobtawVVV1jUxLlryru0+Zx3/dxQFRDhTzGb7bbyw7M49ZPHVEN8uqOjLTONdfSnKN7r7M0jVHqKo/SXKjTAeyv72cz5KTU62SM2AX2pU1X7eCxbwi09GoM7Jpxqc1cuvuvtmmy2+sqr9ZWWvWw6uq6odHrHO0x4yLf5ILJwI5N8m/y/KT1MAl0t2fzzRxw8blT2eNFvFlLVQu2nvhglx0evjtL1j1C5mW2bhVkrMyzfZ6ypI1R6iq53X3Q5LcL8nvZZogcO1nQhXALvSBTDPZre26FSzm6O6+16obsaALqur63f2JJKmq62Vct7m1Mq9vtHGQ579V1TeTnD9fXmqdo40PsmMzDd5+RaYvCsdnWrAcDghV9bbuPm6P10liHTAOPM9OclpVbaw1+KOZZilc0mUzHZg4o7vP39+NDyK3qqprJvl0pkk4DgmHfBfEPdatuHmmLyRruW4Fy6iqk5I8rbvfv+q2LKGq7pZpJrJPzpuOSfLw7t7bwr5sQVU9P8lbM82W+eFBNd+a5D7dfc58+cgkr+7uO138/wRgT/Ps2cfNF0/p7vessj0Hq6p6dKZx5tdN8nebr8oaz7gsgE0zH1aSJ+eis2RVkid3921X0jAOePO6Y53pTPINMgWUb+bCN421mHK3qn4s06K2x2Q6ynf7JL/a3Weusl0Hs6q6a6auJHfMNPb0zEwf4E9dsOZHk9y0u785X75Mkvd197ot/g7AQaaqnt7dj1x1O0Y55APYhkNh3Qq21+i1zlZl0/pmxyX5/zIttvk/HJz43oxez6WqfjXTujWbu8y8uLt/e6maAMB3OuQD2OZ1K5J8YtNVRyZ5e3cfv5KGwQFi00LTv53k/d39wo1tq27bwWov67m8bcR6LnOXmTvOF9+qywwAjCeAVV05yVVzaKxbAZdYVb0qyeeS3CPJLTOtHXX6HjMjcglU1e9lmsnqm5kWKH1rpimMD/r1XACAi3fIBzDg4lXV5ZPcK9PZr49V1fcn+cHuft2Km3bQW9f1XACAfRPAAAbby3oup2SahOONq2wXALA864ABjLeu67kAAPvhDBgAAMAgl1p1AwAAAA4VAhgAAMAgAhgAAMAgAhgAAMAgAhgAAMAg/z8XYdJcGCSwXwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1080x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bo3bTTafzR-V",
        "outputId": "4ea3714a-c5da-4b2d-bc1f-f4fa10036fd3"
      },
      "source": [
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.stem import PorterStemmer\n",
        "import pandas as pd\n",
        "nltk.download('wordnet')\n",
        "\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "ps = PorterStemmer()\n",
        "\n",
        "def Lemmatize(text):\n",
        "  lemma_words = [[x, lemmatizer.lemmatize(x).lower()] for x in text.split(\" \")]\n",
        "  return lemma_words\n",
        "\n",
        "def Stemmed(text):\n",
        "  stemmed_words = [[x, ps.stem(x)] for x in text.split(\" \")]\n",
        "  return stemmed_words"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AZUsJXvh0D7U",
        "outputId": "66d5245d-68fa-4b24-e28d-a3ea3611c7f2"
      },
      "source": [
        "import re\n",
        "\n",
        "user_input = input()\n",
        "# Removing all punctuations and special characters from the string as they might \n",
        "# hinder the process of Stemming and Lemmatization\n",
        "user_input = re.sub('[^a-zA-Z0-9\\s]', \"\", user_input)\n",
        "lemma_words = Lemmatize(user_input)\n",
        "stemmed_words = Stemmed(user_input)\n",
        "df_lst = []\n",
        "print(\"Word\\tLemma\\tStem\")\n",
        "for (word, lemma), (word_x, stemmed) in zip(lemma_words, stemmed_words):\n",
        "  print(word, '->', lemma, '->', stemmed)\n",
        "  df_lst.append([word, lemma, stemmed])\n",
        "\n",
        "df = pd.DataFrame(df_lst, columns=[\"Original Word\", \"Lemmatized Form\", \"Stemmed Form\"])\n",
        "df.to_csv(\"word_lemma_stem.csv\", index=False)"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are many variations of passages of Lorem Ipsum available, but the majority have suffered alteration in some form, by injected humour, or randomised words which don't look even slightly believable. If you are going to use a passage of Lorem Ipsum, you need to be sure there isn't anything embarrassing hidden in the middle of text. All the Lorem Ipsum generators on the Internet tend to repeat predefined chunks as necessary, making this the first true generator on the Internet. It uses a dictionary of over 200 Latin words, combined with a handful of model sentence structures, to generate Lorem Ipsum which looks reasonable. The generated Lorem Ipsum is therefore always free from repetition, injected humour, or non-characteristic words etc.\n",
            "Word\tLemma\tStem\n",
            "There -> there -> there\n",
            "are -> are -> are\n",
            "many -> many -> mani\n",
            "variations -> variation -> variat\n",
            "of -> of -> of\n",
            "passages -> passage -> passag\n",
            "of -> of -> of\n",
            "Lorem -> lorem -> lorem\n",
            "Ipsum -> ipsum -> ipsum\n",
            "available -> available -> avail\n",
            "but -> but -> but\n",
            "the -> the -> the\n",
            "majority -> majority -> major\n",
            "have -> have -> have\n",
            "suffered -> suffered -> suffer\n",
            "alteration -> alteration -> alter\n",
            "in -> in -> in\n",
            "some -> some -> some\n",
            "form -> form -> form\n",
            "by -> by -> by\n",
            "injected -> injected -> inject\n",
            "humour -> humour -> humour\n",
            "or -> or -> or\n",
            "randomised -> randomised -> randomis\n",
            "words -> word -> word\n",
            "which -> which -> which\n",
            "dont -> dont -> dont\n",
            "look -> look -> look\n",
            "even -> even -> even\n",
            "slightly -> slightly -> slightli\n",
            "believable -> believable -> believ\n",
            "If -> if -> If\n",
            "you -> you -> you\n",
            "are -> are -> are\n",
            "going -> going -> go\n",
            "to -> to -> to\n",
            "use -> use -> use\n",
            "a -> a -> a\n",
            "passage -> passage -> passag\n",
            "of -> of -> of\n",
            "Lorem -> lorem -> lorem\n",
            "Ipsum -> ipsum -> ipsum\n",
            "you -> you -> you\n",
            "need -> need -> need\n",
            "to -> to -> to\n",
            "be -> be -> be\n",
            "sure -> sure -> sure\n",
            "there -> there -> there\n",
            "isnt -> isnt -> isnt\n",
            "anything -> anything -> anyth\n",
            "embarrassing -> embarrassing -> embarrass\n",
            "hidden -> hidden -> hidden\n",
            "in -> in -> in\n",
            "the -> the -> the\n",
            "middle -> middle -> middl\n",
            "of -> of -> of\n",
            "text -> text -> text\n",
            "All -> all -> all\n",
            "the -> the -> the\n",
            "Lorem -> lorem -> lorem\n",
            "Ipsum -> ipsum -> ipsum\n",
            "generators -> generator -> gener\n",
            "on -> on -> on\n",
            "the -> the -> the\n",
            "Internet -> internet -> internet\n",
            "tend -> tend -> tend\n",
            "to -> to -> to\n",
            "repeat -> repeat -> repeat\n",
            "predefined -> predefined -> predefin\n",
            "chunks -> chunk -> chunk\n",
            "as -> a -> as\n",
            "necessary -> necessary -> necessari\n",
            "making -> making -> make\n",
            "this -> this -> thi\n",
            "the -> the -> the\n",
            "first -> first -> first\n",
            "true -> true -> true\n",
            "generator -> generator -> gener\n",
            "on -> on -> on\n",
            "the -> the -> the\n",
            "Internet -> internet -> internet\n",
            "It -> it -> It\n",
            "uses -> us -> use\n",
            "a -> a -> a\n",
            "dictionary -> dictionary -> dictionari\n",
            "of -> of -> of\n",
            "over -> over -> over\n",
            "200 -> 200 -> 200\n",
            "Latin -> latin -> latin\n",
            "words -> word -> word\n",
            "combined -> combined -> combin\n",
            "with -> with -> with\n",
            "a -> a -> a\n",
            "handful -> handful -> hand\n",
            "of -> of -> of\n",
            "model -> model -> model\n",
            "sentence -> sentence -> sentenc\n",
            "structures -> structure -> structur\n",
            "to -> to -> to\n",
            "generate -> generate -> gener\n",
            "Lorem -> lorem -> lorem\n",
            "Ipsum -> ipsum -> ipsum\n",
            "which -> which -> which\n",
            "looks -> look -> look\n",
            "reasonable -> reasonable -> reason\n",
            "The -> the -> the\n",
            "generated -> generated -> gener\n",
            "Lorem -> lorem -> lorem\n",
            "Ipsum -> ipsum -> ipsum\n",
            "is -> is -> is\n",
            "therefore -> therefore -> therefor\n",
            "always -> always -> alway\n",
            "free -> free -> free\n",
            "from -> from -> from\n",
            "repetition -> repetition -> repetit\n",
            "injected -> injected -> inject\n",
            "humour -> humour -> humour\n",
            "or -> or -> or\n",
            "noncharacteristic -> noncharacteristic -> noncharacterist\n",
            "words -> word -> word\n",
            "etc -> etc -> etc\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "id": "j0a_S33o16S-",
        "outputId": "b683a2d8-1fac-4cb9-dc3d-58b0747dda14"
      },
      "source": [
        "df.head(10)"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Original Word</th>\n",
              "      <th>Lemmatized Form</th>\n",
              "      <th>Stemmed Form</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>There</td>\n",
              "      <td>there</td>\n",
              "      <td>there</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>are</td>\n",
              "      <td>are</td>\n",
              "      <td>are</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>many</td>\n",
              "      <td>many</td>\n",
              "      <td>mani</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>variations</td>\n",
              "      <td>variation</td>\n",
              "      <td>variat</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>of</td>\n",
              "      <td>of</td>\n",
              "      <td>of</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>passages</td>\n",
              "      <td>passage</td>\n",
              "      <td>passag</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>of</td>\n",
              "      <td>of</td>\n",
              "      <td>of</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Lorem</td>\n",
              "      <td>lorem</td>\n",
              "      <td>lorem</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Ipsum</td>\n",
              "      <td>ipsum</td>\n",
              "      <td>ipsum</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>available</td>\n",
              "      <td>available</td>\n",
              "      <td>avail</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  Original Word Lemmatized Form Stemmed Form\n",
              "0         There           there        there\n",
              "1           are             are          are\n",
              "2          many            many         mani\n",
              "3    variations       variation       variat\n",
              "4            of              of           of\n",
              "5      passages         passage       passag\n",
              "6            of              of           of\n",
              "7         Lorem           lorem        lorem\n",
              "8         Ipsum           ipsum        ipsum\n",
              "9     available       available        avail"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    }
  ]
}