Grammatical_Error_Correction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Grammatical Error Correction.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "bbc4474c843543ceb43dd335f3aecfb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7a3ded7900d54fb0ac6dfad5a18e752e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c1dab40291c84d368ade0236d95f4cef",
              "IPY_MODEL_d170d5ebd4d4405a96ac85c23a1e4d6d"
            ]
          }
        },
        "7a3ded7900d54fb0ac6dfad5a18e752e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c1dab40291c84d368ade0236d95f4cef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3f7ff21318674b5b81cf84e2aed6793b",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 57152,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 57152,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2aec8c1fc74b4200b336a93367dc5b18"
          }
        },
        "d170d5ebd4d4405a96ac85c23a1e4d6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5d9c80f8dbb74aabaf8c6a01d20c8ef6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 57152/57152 [09:01&lt;00:00, 105.46it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_65d18e8ff1744a19b522379d283d16fb"
          }
        },
        "3f7ff21318674b5b81cf84e2aed6793b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2aec8c1fc74b4200b336a93367dc5b18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5d9c80f8dbb74aabaf8c6a01d20c8ef6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "65d18e8ff1744a19b522379d283d16fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8c26d2ee6f4242f4a9f48aaf45b25acf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_70af676e709d4943b4726b852d671f90",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_488c7adc69c8457c84f0bc0fd57ecb9a",
              "IPY_MODEL_b0fb8633acd241b1a30f5d8c196e900d"
            ]
          }
        },
        "70af676e709d4943b4726b852d671f90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "488c7adc69c8457c84f0bc0fd57ecb9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bc70964b308d4122ac9fe7a71344cfb4",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1313,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1313,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7cdff92d395147ad8789d6dc025c08c8"
          }
        },
        "b0fb8633acd241b1a30f5d8c196e900d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c8f8627c971a4b72b82d98baf339d9cd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1313/1313 [00:17&lt;00:00, 74.06it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7485f08f3b49470796a0cbe05a53e8f2"
          }
        },
        "bc70964b308d4122ac9fe7a71344cfb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7cdff92d395147ad8789d6dc025c08c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c8f8627c971a4b72b82d98baf339d9cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7485f08f3b49470796a0cbe05a53e8f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fTig2bdx9ANu",
        "colab_type": "text"
      },
      "source": [
        "#Import\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fAuPRezu87Y5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "39f86011-53dc-497e-a207-6ef71dcbad8f"
      },
      "source": [
        "!pip3 install pyinflect\n",
        "import nltk\n",
        "nltk.download('all')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pyinflect in /usr/local/lib/python3.6/dist-packages (0.5.1)\n",
            "[nltk_data] Downloading collection 'all'\n",
            "[nltk_data]    | \n",
            "[nltk_data]    | Downloading package abc to /root/nltk_data...\n",
            "[nltk_data]    |   Package abc is already up-to-date!\n",
            "[nltk_data]    | Downloading package alpino to /root/nltk_data...\n",
            "[nltk_data]    |   Package alpino is already up-to-date!\n",
            "[nltk_data]    | Downloading package biocreative_ppi to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package biocreative_ppi is already up-to-date!\n",
            "[nltk_data]    | Downloading package brown to /root/nltk_data...\n",
            "[nltk_data]    |   Package brown is already up-to-date!\n",
            "[nltk_data]    | Downloading package brown_tei to /root/nltk_data...\n",
            "[nltk_data]    |   Package brown_tei is already up-to-date!\n",
            "[nltk_data]    | Downloading package cess_cat to /root/nltk_data...\n",
            "[nltk_data]    |   Package cess_cat is already up-to-date!\n",
            "[nltk_data]    | Downloading package cess_esp to /root/nltk_data...\n",
            "[nltk_data]    |   Package cess_esp is already up-to-date!\n",
            "[nltk_data]    | Downloading package chat80 to /root/nltk_data...\n",
            "[nltk_data]    |   Package chat80 is already up-to-date!\n",
            "[nltk_data]    | Downloading package city_database to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package city_database is already up-to-date!\n",
            "[nltk_data]    | Downloading package cmudict to /root/nltk_data...\n",
            "[nltk_data]    |   Package cmudict is already up-to-date!\n",
            "[nltk_data]    | Downloading package comparative_sentences to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package comparative_sentences is already up-to-\n",
            "[nltk_data]    |       date!\n",
            "[nltk_data]    | Downloading package comtrans to /root/nltk_data...\n",
            "[nltk_data]    |   Package comtrans is already up-to-date!\n",
            "[nltk_data]    | Downloading package conll2000 to /root/nltk_data...\n",
            "[nltk_data]    |   Package conll2000 is already up-to-date!\n",
            "[nltk_data]    | Downloading package conll2002 to /root/nltk_data...\n",
            "[nltk_data]    |   Package conll2002 is already up-to-date!\n",
            "[nltk_data]    | Downloading package conll2007 to /root/nltk_data...\n",
            "[nltk_data]    |   Package conll2007 is already up-to-date!\n",
            "[nltk_data]    | Downloading package crubadan to /root/nltk_data...\n",
            "[nltk_data]    |   Package crubadan is already up-to-date!\n",
            "[nltk_data]    | Downloading package dependency_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package dependency_treebank is already up-to-date!\n",
            "[nltk_data]    | Downloading package dolch to /root/nltk_data...\n",
            "[nltk_data]    |   Package dolch is already up-to-date!\n",
            "[nltk_data]    | Downloading package europarl_raw to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package europarl_raw is already up-to-date!\n",
            "[nltk_data]    | Downloading package floresta to /root/nltk_data...\n",
            "[nltk_data]    |   Package floresta is already up-to-date!\n",
            "[nltk_data]    | Downloading package framenet_v15 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package framenet_v15 is already up-to-date!\n",
            "[nltk_data]    | Downloading package framenet_v17 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package framenet_v17 is already up-to-date!\n",
            "[nltk_data]    | Downloading package gazetteers to /root/nltk_data...\n",
            "[nltk_data]    |   Package gazetteers is already up-to-date!\n",
            "[nltk_data]    | Downloading package genesis to /root/nltk_data...\n",
            "[nltk_data]    |   Package genesis is already up-to-date!\n",
            "[nltk_data]    | Downloading package gutenberg to /root/nltk_data...\n",
            "[nltk_data]    |   Package gutenberg is already up-to-date!\n",
            "[nltk_data]    | Downloading package ieer to /root/nltk_data...\n",
            "[nltk_data]    |   Package ieer is already up-to-date!\n",
            "[nltk_data]    | Downloading package inaugural to /root/nltk_data...\n",
            "[nltk_data]    |   Package inaugural is already up-to-date!\n",
            "[nltk_data]    | Downloading package indian to /root/nltk_data...\n",
            "[nltk_data]    |   Package indian is already up-to-date!\n",
            "[nltk_data]    | Downloading package jeita to /root/nltk_data...\n",
            "[nltk_data]    |   Package jeita is already up-to-date!\n",
            "[nltk_data]    | Downloading package kimmo to /root/nltk_data...\n",
            "[nltk_data]    |   Package kimmo is already up-to-date!\n",
            "[nltk_data]    | Downloading package knbc to /root/nltk_data...\n",
            "[nltk_data]    |   Package knbc is already up-to-date!\n",
            "[nltk_data]    | Downloading package lin_thesaurus to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package lin_thesaurus is already up-to-date!\n",
            "[nltk_data]    | Downloading package mac_morpho to /root/nltk_data...\n",
            "[nltk_data]    |   Package mac_morpho is already up-to-date!\n",
            "[nltk_data]    | Downloading package machado to /root/nltk_data...\n",
            "[nltk_data]    |   Package machado is already up-to-date!\n",
            "[nltk_data]    | Downloading package masc_tagged to /root/nltk_data...\n",
            "[nltk_data]    |   Package masc_tagged is already up-to-date!\n",
            "[nltk_data]    | Downloading package moses_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package moses_sample is already up-to-date!\n",
            "[nltk_data]    | Downloading package movie_reviews to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package movie_reviews is already up-to-date!\n",
            "[nltk_data]    | Downloading package names to /root/nltk_data...\n",
            "[nltk_data]    |   Package names is already up-to-date!\n",
            "[nltk_data]    | Downloading package nombank.1.0 to /root/nltk_data...\n",
            "[nltk_data]    |   Package nombank.1.0 is already up-to-date!\n",
            "[nltk_data]    | Downloading package nps_chat to /root/nltk_data...\n",
            "[nltk_data]    |   Package nps_chat is already up-to-date!\n",
            "[nltk_data]    | Downloading package omw to /root/nltk_data...\n",
            "[nltk_data]    |   Package omw is already up-to-date!\n",
            "[nltk_data]    | Downloading package opinion_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package opinion_lexicon is already up-to-date!\n",
            "[nltk_data]    | Downloading package paradigms to /root/nltk_data...\n",
            "[nltk_data]    |   Package paradigms is already up-to-date!\n",
            "[nltk_data]    | Downloading package pil to /root/nltk_data...\n",
            "[nltk_data]    |   Package pil is already up-to-date!\n",
            "[nltk_data]    | Downloading package pl196x to /root/nltk_data...\n",
            "[nltk_data]    |   Package pl196x is already up-to-date!\n",
            "[nltk_data]    | Downloading package ppattach to /root/nltk_data...\n",
            "[nltk_data]    |   Package ppattach is already up-to-date!\n",
            "[nltk_data]    | Downloading package problem_reports to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package problem_reports is already up-to-date!\n",
            "[nltk_data]    | Downloading package propbank to /root/nltk_data...\n",
            "[nltk_data]    |   Package propbank is already up-to-date!\n",
            "[nltk_data]    | Downloading package ptb to /root/nltk_data...\n",
            "[nltk_data]    |   Package ptb is already up-to-date!\n",
            "[nltk_data]    | Downloading package product_reviews_1 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package product_reviews_1 is already up-to-date!\n",
            "[nltk_data]    | Downloading package product_reviews_2 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package product_reviews_2 is already up-to-date!\n",
            "[nltk_data]    | Downloading package pros_cons to /root/nltk_data...\n",
            "[nltk_data]    |   Package pros_cons is already up-to-date!\n",
            "[nltk_data]    | Downloading package qc to /root/nltk_data...\n",
            "[nltk_data]    |   Package qc is already up-to-date!\n",
            "[nltk_data]    | Downloading package reuters to /root/nltk_data...\n",
            "[nltk_data]    |   Package reuters is already up-to-date!\n",
            "[nltk_data]    | Downloading package rte to /root/nltk_data...\n",
            "[nltk_data]    |   Package rte is already up-to-date!\n",
            "[nltk_data]    | Downloading package semcor to /root/nltk_data...\n",
            "[nltk_data]    |   Package semcor is already up-to-date!\n",
            "[nltk_data]    | Downloading package senseval to /root/nltk_data...\n",
            "[nltk_data]    |   Package senseval is already up-to-date!\n",
            "[nltk_data]    | Downloading package sentiwordnet to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package sentiwordnet is already up-to-date!\n",
            "[nltk_data]    | Downloading package sentence_polarity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package sentence_polarity is already up-to-date!\n",
            "[nltk_data]    | Downloading package shakespeare to /root/nltk_data...\n",
            "[nltk_data]    |   Package shakespeare is already up-to-date!\n",
            "[nltk_data]    | Downloading package sinica_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package sinica_treebank is already up-to-date!\n",
            "[nltk_data]    | Downloading package smultron to /root/nltk_data...\n",
            "[nltk_data]    |   Package smultron is already up-to-date!\n",
            "[nltk_data]    | Downloading package state_union to /root/nltk_data...\n",
            "[nltk_data]    |   Package state_union is already up-to-date!\n",
            "[nltk_data]    | Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]    |   Package stopwords is already up-to-date!\n",
            "[nltk_data]    | Downloading package subjectivity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package subjectivity is already up-to-date!\n",
            "[nltk_data]    | Downloading package swadesh to /root/nltk_data...\n",
            "[nltk_data]    |   Package swadesh is already up-to-date!\n",
            "[nltk_data]    | Downloading package switchboard to /root/nltk_data...\n",
            "[nltk_data]    |   Package switchboard is already up-to-date!\n",
            "[nltk_data]    | Downloading package timit to /root/nltk_data...\n",
            "[nltk_data]    |   Package timit is already up-to-date!\n",
            "[nltk_data]    | Downloading package toolbox to /root/nltk_data...\n",
            "[nltk_data]    |   Package toolbox is already up-to-date!\n",
            "[nltk_data]    | Downloading package treebank to /root/nltk_data...\n",
            "[nltk_data]    |   Package treebank is already up-to-date!\n",
            "[nltk_data]    | Downloading package twitter_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package twitter_samples is already up-to-date!\n",
            "[nltk_data]    | Downloading package udhr to /root/nltk_data...\n",
            "[nltk_data]    |   Package udhr is already up-to-date!\n",
            "[nltk_data]    | Downloading package udhr2 to /root/nltk_data...\n",
            "[nltk_data]    |   Package udhr2 is already up-to-date!\n",
            "[nltk_data]    | Downloading package unicode_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package unicode_samples is already up-to-date!\n",
            "[nltk_data]    | Downloading package universal_treebanks_v20 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package universal_treebanks_v20 is already up-to-\n",
            "[nltk_data]    |       date!\n",
            "[nltk_data]    | Downloading package verbnet to /root/nltk_data...\n",
            "[nltk_data]    |   Package verbnet is already up-to-date!\n",
            "[nltk_data]    | Downloading package verbnet3 to /root/nltk_data...\n",
            "[nltk_data]    |   Package verbnet3 is already up-to-date!\n",
            "[nltk_data]    | Downloading package webtext to /root/nltk_data...\n",
            "[nltk_data]    |   Package webtext is already up-to-date!\n",
            "[nltk_data]    | Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]    |   Package wordnet is already up-to-date!\n",
            "[nltk_data]    | Downloading package wordnet_ic to /root/nltk_data...\n",
            "[nltk_data]    |   Package wordnet_ic is already up-to-date!\n",
            "[nltk_data]    | Downloading package words to /root/nltk_data...\n",
            "[nltk_data]    |   Package words is already up-to-date!\n",
            "[nltk_data]    | Downloading package ycoe to /root/nltk_data...\n",
            "[nltk_data]    |   Package ycoe is already up-to-date!\n",
            "[nltk_data]    | Downloading package rslp to /root/nltk_data...\n",
            "[nltk_data]    |   Package rslp is already up-to-date!\n",
            "[nltk_data]    | Downloading package maxent_treebank_pos_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package maxent_treebank_pos_tagger is already up-\n",
            "[nltk_data]    |       to-date!\n",
            "[nltk_data]    | Downloading package universal_tagset to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package universal_tagset is already up-to-date!\n",
            "[nltk_data]    | Downloading package maxent_ne_chunker to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package maxent_ne_chunker is already up-to-date!\n",
            "[nltk_data]    | Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]    |   Package punkt is already up-to-date!\n",
            "[nltk_data]    | Downloading package book_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package book_grammars is already up-to-date!\n",
            "[nltk_data]    | Downloading package sample_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package sample_grammars is already up-to-date!\n",
            "[nltk_data]    | Downloading package spanish_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package spanish_grammars is already up-to-date!\n",
            "[nltk_data]    | Downloading package basque_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package basque_grammars is already up-to-date!\n",
            "[nltk_data]    | Downloading package large_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package large_grammars is already up-to-date!\n",
            "[nltk_data]    | Downloading package tagsets to /root/nltk_data...\n",
            "[nltk_data]    |   Package tagsets is already up-to-date!\n",
            "[nltk_data]    | Downloading package snowball_data to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package snowball_data is already up-to-date!\n",
            "[nltk_data]    | Downloading package bllip_wsj_no_aux to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package bllip_wsj_no_aux is already up-to-date!\n",
            "[nltk_data]    | Downloading package word2vec_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package word2vec_sample is already up-to-date!\n",
            "[nltk_data]    | Downloading package panlex_swadesh to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package panlex_swadesh is already up-to-date!\n",
            "[nltk_data]    | Downloading package mte_teip5 to /root/nltk_data...\n",
            "[nltk_data]    |   Package mte_teip5 is already up-to-date!\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package averaged_perceptron_tagger is already up-\n",
            "[nltk_data]    |       to-date!\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger_ru to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package averaged_perceptron_tagger_ru is already\n",
            "[nltk_data]    |       up-to-date!\n",
            "[nltk_data]    | Downloading package perluniprops to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package perluniprops is already up-to-date!\n",
            "[nltk_data]    | Downloading package nonbreaking_prefixes to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package nonbreaking_prefixes is already up-to-date!\n",
            "[nltk_data]    | Downloading package vader_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Package vader_lexicon is already up-to-date!\n",
            "[nltk_data]    | Downloading package porter_test to /root/nltk_data...\n",
            "[nltk_data]    |   Package porter_test is already up-to-date!\n",
            "[nltk_data]    | Downloading package wmt15_eval to /root/nltk_data...\n",
            "[nltk_data]    |   Package wmt15_eval is already up-to-date!\n",
            "[nltk_data]    | Downloading package mwa_ppdb to /root/nltk_data...\n",
            "[nltk_data]    |   Package mwa_ppdb is already up-to-date!\n",
            "[nltk_data]    | \n",
            "[nltk_data]  Done downloading collection all\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0EhwLbfk8-0I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.stem import WordNetLemmatizer  \n",
        "lemmatizer = WordNetLemmatizer() \n",
        "from pyinflect import getAllInflections, getInflection\n",
        "from nltk import word_tokenize\n",
        "import spacy\n",
        "import re\n",
        "import spacy\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "import csv\n",
        "import json"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4qSX7ewn9Frn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "31e3ba1f-30c6-48ed-a200-99afa68f1a28"
      },
      "source": [
        "!pip install autocorrect\n",
        "from autocorrect import Speller\n",
        "spell = Speller(lang='en')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: autocorrect in /usr/local/lib/python3.6/dist-packages (2.0.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y-yD7s4w9I-P",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "fb0af884-723d-4728-a185-900d1869bdae"
      },
      "source": [
        "!git clone https://github.com/nozomiyamada/contest2"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'contest2' already exists and is not an empty directory.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LhAomErf9NVe",
        "colab_type": "text"
      },
      "source": [
        "#Function for generating candidate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jXwi9_kO9NtR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#noun.csv\n",
        "noun_dict2 = dict()\n",
        "noun_list = []\n",
        "with open('/content/contest2/noun.json') as csv_file:\n",
        "  csv_reader = csv.reader(csv_file, delimiter=',')\n",
        "  for row in csv_reader:\n",
        "    for string in row:\n",
        "      noun_list.append(string.strip(' ').strip(\"]\").strip(\"[\").strip(' \" '))\n",
        "for i in range(len(noun_list)-1):\n",
        "  noun_dict2[noun_list[i]] = noun_list[i : i+2]\n",
        "\n",
        "#verb.csv\n",
        "verb_dict2 = dict()\n",
        "with open('/content/contest2/verbs-dictionaries.csv') as csv_file:\n",
        "  csv_reader = csv.reader(csv_file,delimiter =',')\n",
        "  for row in csv_reader:\n",
        "    for verbs in row:\n",
        "      verbs_list = verbs.split('\\t')\n",
        "      verb_dict2[verbs_list[0]] = verbs_list[:]\n",
        "\n",
        "#prep.json\n",
        "prep_list = []\n",
        "with open('/content/contest2/prep.json') as f:\n",
        "  data = json.load(f)\n",
        "  #prep_list.append(\"\")\n",
        "  for prep in data:\n",
        "    prep_list.append(prep)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8wkk-TyW9Wp2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#find candidates\n",
        "\n",
        "def pos_tag(sentence):\n",
        "   tokens = word_tokenize(sentence)\n",
        "   return nltk.pos_tag(tokens)\n",
        "\n",
        "def lemma(word):\n",
        "  nlp = spacy.load(\"en_core_web_sm\")\n",
        "  doc = nlp(word)\n",
        "  lemma_word = [token.lemma_ for token in doc]\n",
        "  return lemma_word\n",
        "\n",
        "def verb_form_candidate(word):\n",
        "  infection_dict = getAllInflections(word , 'V')\n",
        "  if word in verb_dict2:\n",
        "    get_infection = verb_dict2[word]\n",
        "  else:\n",
        "    get_infection = [tup[0] for tup in infection_dict.values()]\n",
        "    get_infection.append(word)\n",
        "  return get_infection\n",
        "\n",
        "def be_verb():\n",
        "  return ['be', 'is', 'am', 'are','was', 'were', 'been','being']\n",
        "\n",
        "def article_candidate():\n",
        "  return ['an','a','the',\"some\",'any','this','that','those','these', '']\n",
        "\n",
        "def  noun_candidate(word):\n",
        "  noun_dict1 = getAllInflections(word , 'N')\n",
        "  if len(word) > 1 and word in noun_dict2:\n",
        "    get_forms = noun_dict2[word]\n",
        "  else:\n",
        "    get_forms = [tup[0] for tup in noun_dict1.values()]\n",
        "    if len(get_forms) == 0:\n",
        "      get_forms.append(word)\n",
        "      get_forms.append(word + 's')\n",
        "      if word.endswith('y') and len(word) > 1 and word[-2] not in ['a','e','i','o','u']:\n",
        "        get_forms.append(word[ : -1] + 'ies')\n",
        "  return get_forms\n",
        "\n",
        "def prep_candidate():\n",
        "   return prep_list"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NZl_Q1dJ9ZmG",
        "colab_type": "text"
      },
      "source": [
        "#LM Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hSC4unX39btQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 218
        },
        "outputId": "1dd84629-23a4-493d-ed8a-4ab3f1f922ea"
      },
      "source": [
        "#Install KenLM Library\n",
        "!wget -O - https://kheafield.com/code/kenlm.tar.gz |tar xz\n",
        "!mkdir kenlm/build"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-08-18 12:58:25--  https://kheafield.com/code/kenlm.tar.gz\n",
            "Resolving kheafield.com (kheafield.com)... 35.196.63.85\n",
            "Connecting to kheafield.com (kheafield.com)|35.196.63.85|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 490441 (479K) [application/x-gzip]\n",
            "Saving to: ‘STDOUT’\n",
            "\n",
            "-                   100%[===================>] 478.95K   516KB/s    in 0.9s    \n",
            "\n",
            "2020-08-18 12:58:27 (516 KB/s) - written to stdout [490441/490441]\n",
            "\n",
            "mkdir: cannot create directory ‘kenlm/build’: File exists\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-U2cFZLN9fIb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 622
        },
        "outputId": "1a1338f3-8882-4d43-e26c-9038c07da6f8"
      },
      "source": [
        "%cd kenlm/build\n",
        "!cmake ..\n",
        "!make -j2"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/kenlm/build\n",
            "-- Boost version: 1.65.1\n",
            "-- Found the following Boost libraries:\n",
            "--   program_options\n",
            "--   system\n",
            "--   thread\n",
            "--   unit_test_framework\n",
            "--   chrono\n",
            "--   date_time\n",
            "--   atomic\n",
            "-- Could NOT find Eigen3 (missing: EIGEN3_INCLUDE_DIR EIGEN3_VERSION_OK) (Required is at least version \"2.91.0\")\n",
            "CMake Warning at lm/interpolate/CMakeLists.txt:65 (message):\n",
            "  Not building interpolation.  Eigen3 was not found.\n",
            "\n",
            "\n",
            "-- To install Eigen3 in your home directory, copy paste this:\n",
            "export EIGEN3_ROOT=$HOME/eigen-eigen-07105f7124f9\n",
            "(cd $HOME; wget -O - https://bitbucket.org/eigen/eigen/get/3.2.8.tar.bz2 |tar xj)\n",
            "rm CMakeCache.txt\n",
            "\n",
            "-- Configuring done\n",
            "-- Generating done\n",
            "-- Build files have been written to: /content/kenlm/build\n",
            "[  5%] Built target kenlm_filter\n",
            "[ 43%] Built target kenlm_util\n",
            "[ 68%] Built target kenlm\n",
            "[ 71%] Built target probing_hash_table_benchmark\n",
            "[ 73%] Built target fragment\n",
            "[ 76%] Built target build_binary\n",
            "[ 78%] Built target kenlm_benchmark\n",
            "[ 81%] Built target query\n",
            "[ 90%] Built target kenlm_builder\n",
            "[ 92%] Built target phrase_table_vocab\n",
            "[ 95%] Built target filter\n",
            "[ 97%] Built target count_ngrams\n",
            "[100%] Built target lmplz\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xm-wdNqB9ig0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "863c4063-868f-444a-f29f-fdc31803a715"
      },
      "source": [
        "#Dowload news data \n",
        "!gdown --id 1tGt5FJIZ0RPKAz4NxxhL1PvNd85AJBQu"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1tGt5FJIZ0RPKAz4NxxhL1PvNd85AJBQu\n",
            "To: /content/kenlm/build/gigaword3_nyt_eng.tar.gz\n",
            "2.54GB [00:21, 116MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HAqZBMbI9k2K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!tar -xf gigaword3_nyt_eng.tar.gz"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1LdyY2M79l5z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "0cab9f4a-85c0-4523-c46f-4e3c1c0a342f"
      },
      "source": [
        "#Dowload gutenberg\n",
        "!gdown --id 1PeMJ8Z9HBDravgGzZ1baDRnliHRf1O75"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1PeMJ8Z9HBDravgGzZ1baDRnliHRf1O75\n",
            "To: /content/kenlm/build/Gutenberg.zip\n",
            "462MB [00:02, 196MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oo12k4by9pV4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from zipfile import ZipFile\n",
        "with ZipFile('/content/kenlm/build/Gutenberg.zip', 'r') as zipObj:\n",
        "   # Extract all the contents of zip file in current directory\n",
        "   zipObj.extractall()"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XV2riwom9t-e",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 958
        },
        "outputId": "da5d6909-0c9f-4c5c-9fa4-285b3e7677c2"
      },
      "source": [
        "ls #/content/kenlm/build/kenlm/build/kenlm"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " \u001b[0m\u001b[01;34mbin\u001b[0m/                                      nyt_eng_199801   nyt_eng_200209\n",
            " CMakeCache.txt                            nyt_eng_199802   nyt_eng_200210\n",
            " \u001b[01;34mCMakeFiles\u001b[0m/                               nyt_eng_199803   nyt_eng_200211\n",
            " cmake_install.cmake                       nyt_eng_199804   nyt_eng_200212\n",
            "'decompose_n_add_token[i]_in_J.txt'        nyt_eng_199805   nyt_eng_200301\n",
            " file_for_evaluate.txt                     nyt_eng_199806   nyt_eng_200302\n",
            " gec-test-set.txt                          nyt_eng_199807   nyt_eng_200303\n",
            " gigaword3_nyt_eng.tar.gz                  nyt_eng_199808   nyt_eng_200304\n",
            " \u001b[01;34mGutenberg\u001b[0m/                                nyt_eng_199809   nyt_eng_200305\n",
            " Gutenberg.zip                             nyt_eng_199810   nyt_eng_200306\n",
            " \u001b[01;34mlib\u001b[0m/                                      nyt_eng_199811   nyt_eng_200307\n",
            " \u001b[01;34mlm\u001b[0m/                                       nyt_eng_199812   nyt_eng_200308\n",
            " Makefile                                  nyt_eng_199901   nyt_eng_200309\n",
            "'notdecompose_but_add_token[i]_in_J.txt'   nyt_eng_199902   nyt_eng_200310\n",
            " nyt_eng_199407                            nyt_eng_199903   nyt_eng_200311\n",
            " nyt_eng_199408                            nyt_eng_199904   nyt_eng_200312\n",
            " nyt_eng_199409                            nyt_eng_199905   nyt_eng_200401\n",
            " nyt_eng_199410                            nyt_eng_199906   nyt_eng_200402\n",
            " nyt_eng_199411                            nyt_eng_199907   nyt_eng_200403\n",
            " nyt_eng_199412                            nyt_eng_199908   nyt_eng_200404\n",
            " nyt_eng_199501                            nyt_eng_199909   nyt_eng_200405\n",
            " nyt_eng_199502                            nyt_eng_199910   nyt_eng_200407\n",
            " nyt_eng_199503                            nyt_eng_199911   nyt_eng_200408\n",
            " nyt_eng_199504                            nyt_eng_199912   nyt_eng_200409\n",
            " nyt_eng_199505                            nyt_eng_200001   nyt_eng_200410\n",
            " nyt_eng_199506                            nyt_eng_200002   nyt_eng_200411\n",
            " nyt_eng_199507                            nyt_eng_200003   nyt_eng_200412\n",
            " nyt_eng_199508                            nyt_eng_200004   nyt_eng_200501\n",
            " nyt_eng_199509                            nyt_eng_200005   nyt_eng_200502\n",
            " nyt_eng_199510                            nyt_eng_200006   nyt_eng_200503\n",
            " nyt_eng_199511                            nyt_eng_200007   nyt_eng_200504\n",
            " nyt_eng_199512                            nyt_eng_200008   nyt_eng_200505\n",
            " nyt_eng_199601                            nyt_eng_200009   nyt_eng_200506\n",
            " nyt_eng_199602                            nyt_eng_200010   nyt_eng_200507\n",
            " nyt_eng_199603                            nyt_eng_200011   nyt_eng_200508\n",
            " nyt_eng_199604                            nyt_eng_200012   nyt_eng_200509\n",
            " nyt_eng_199605                            nyt_eng_200101   nyt_eng_200510\n",
            " nyt_eng_199606                            nyt_eng_200102   nyt_eng_200511\n",
            " nyt_eng_199607                            nyt_eng_200103   nyt_eng_200512\n",
            " nyt_eng_199608                            nyt_eng_200104   nyt_eng_200601\n",
            " nyt_eng_199609                            nyt_eng_200105   nyt_eng_200602\n",
            " nyt_eng_199610                            nyt_eng_200106   nyt_eng_200603\n",
            " nyt_eng_199611                            nyt_eng_200107   nyt_eng_200604\n",
            " nyt_eng_199612                            nyt_eng_200108   nyt_eng_200605\n",
            " nyt_eng_199701                            nyt_eng_200109   nyt_eng_200606\n",
            " nyt_eng_199702                            nyt_eng_200110   nyt_eng_200607\n",
            " nyt_eng_199703                            nyt_eng_200111   nyt_eng_200608\n",
            " nyt_eng_199704                            nyt_eng_200112   nyt_eng_200609\n",
            " nyt_eng_199705                            nyt_eng_200201   nyt_eng_200610\n",
            " nyt_eng_199706                            nyt_eng_200202   nyt_eng_200611\n",
            " nyt_eng_199707                            nyt_eng_200203   nyt_eng_200612\n",
            " nyt_eng_199708                            nyt_eng_200204   nyt_eng.txt\n",
            " nyt_eng_199709                            nyt_eng_200205   sixgram.arpa\n",
            " nyt_eng_199710                            nyt_eng_200206   \u001b[01;34mutil\u001b[0m/\n",
            " nyt_eng_199711                            nyt_eng_200207\n",
            " nyt_eng_199712                            nyt_eng_200208\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zBT5u1h09wlf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cat nyt_eng_2001* nyt_eng_2006*  > nyt_eng.txt"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LoIxO8qN9zBB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 171
        },
        "outputId": "7f07bd9c-9c90-44b9-afd6-2f2f04c40b90"
      },
      "source": [
        "#Install KenLM in Python\n",
        "!pip install https://github.com/kpu/kenlm/archive/master.zip"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting https://github.com/kpu/kenlm/archive/master.zip\n",
            "  Using cached https://github.com/kpu/kenlm/archive/master.zip\n",
            "Requirement already satisfied (use --upgrade to upgrade): kenlm==0.0.0 from https://github.com/kpu/kenlm/archive/master.zip in /usr/local/lib/python3.6/dist-packages\n",
            "Building wheels for collected packages: kenlm\n",
            "  Building wheel for kenlm (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for kenlm: filename=kenlm-0.0.0-cp36-cp36m-linux_x86_64.whl size=2330484 sha256=cc96248c35c79d2df703e31d84435d5206980a0d50fd4a6ce772b018e1506f17\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-wai1fi81/wheels/2d/32/73/e3093c9d11dc8abf79c156a4db1a1c5631428059d4f9ff2cba\n",
            "Successfully built kenlm\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eaw04iOH92z3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 793
        },
        "outputId": "8b72acbf-d1dd-459e-c96f-fef7f8bd3316"
      },
      "source": [
        "#Train LM with KenLM\n",
        "!bin/lmplz -o 6 --text nyt_eng.txt --arpa sixgram.arpa"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "=== 1/5 Counting and sorting n-grams ===\n",
            "Reading /content/kenlm/build/nyt_eng.txt\n",
            "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
            "tcmalloc: large alloc 1720688640 bytes == 0x557b808f6000 @  0x7fbfd2f791e7 0x557b7f9e8772 0x557b7f97c358 0x557b7f95b290 0x557b7f947096 0x7fbfd1112b97 0x557b7f948ada\n",
            "tcmalloc: large alloc 9176997888 bytes == 0x557be71f0000 @  0x7fbfd2f791e7 0x557b7f9e8772 0x557b7f9d27aa 0x557b7f9d31c8 0x557b7f95b2ad 0x557b7f947096 0x7fbfd1112b97 0x557b7f948ada\n",
            "****************************************************************************************************\n",
            "Unigram tokens 209075440 types 880985\n",
            "=== 2/5 Calculating and sorting adjusted counts ===\n",
            "Chain sizes: 1:10571820 2:663265024 3:1243621888 4:1989795072 5:2901784576 6:3979590144\n",
            "tcmalloc: large alloc 3979591680 bytes == 0x557b808f6000 @  0x7fbfd2f791e7 0x557b7f9e8772 0x557b7f9d27aa 0x557b7f9d31c8 0x557b7f95b84e 0x557b7f947096 0x7fbfd1112b97 0x557b7f948ada\n",
            "tcmalloc: large alloc 1989795840 bytes == 0x557ce00de000 @  0x7fbfd2f791e7 0x557b7f9e8772 0x557b7f9d27aa 0x557b7f9d31c8 0x557b7f95bc3d 0x557b7f947096 0x7fbfd1112b97 0x557b7f948ada\n",
            "tcmalloc: large alloc 2901786624 bytes == 0x557e0aa72000 @  0x7fbfd2f791e7 0x557b7f9e8772 0x557b7f9d27aa 0x557b7f9d31c8 0x557b7f95bc3d 0x557b7f947096 0x7fbfd1112b97 0x557b7f948ada\n",
            "Statistics:\n",
            "1 880985 D1=0.680597 D2=0.997432 D3+=1.31674\n",
            "2 15721743 D1=0.722756 D2=1.08151 D3+=1.38691\n",
            "3 60179756 D1=0.80287 D2=1.14142 D3+=1.35163\n",
            "4 104114180 D1=0.874009 D2=1.24055 D3+=1.33935\n",
            "5 120994436 D1=0.924204 D2=1.37909 D3+=1.28531\n",
            "6 118342352 D1=0.806938 D2=1.6065 D3+=0.878681\n",
            "Memory estimate for binary LM:\n",
            "type       MB\n",
            "probing  8942 assuming -p 1.5\n",
            "probing 10668 assuming -r models -p 1.5\n",
            "trie     4684 without quantization\n",
            "trie     2673 assuming -q 8 -b 8 quantization \n",
            "trie     3979 assuming -a 22 array pointer compression\n",
            "trie     1968 assuming -a 22 -q 8 -b 8 array pointer compression and quantization\n",
            "=== 3/5 Calculating and sorting initial probabilities ===\n",
            "tcmalloc: large alloc 2498748416 bytes == 0x557e12a72000 @  0x7fbfd2f791e7 0x557b7f9e8772 0x557b7f966232 0x557b7f9671cd 0x7fbfd28acbcd 0x7fbfd26836db 0x7fbfd1212a3f\n",
            "tcmalloc: large alloc 3387850752 bytes == 0x557b808f6000 @  0x7fbfd2f791e7 0x557b7f9e8772 0x557b7f966232 0x557b7f9671cd 0x7fbfd28acbcd 0x7fbfd26836db 0x7fbfd1212a3f\n",
            "tcmalloc: large alloc 3786956800 bytes == 0x557b808f6000 @  0x7fbfd2f791e7 0x557b7f9e8772 0x557b7f966232 0x557b7f9671cd 0x7fbfd28acbcd 0x7fbfd26836db 0x7fbfd1212a3f\n",
            "Chain sizes: 1:10571820 2:251547888 3:1203595120 4:2306512128 5:3363663616 6:3786955264\n",
            "tcmalloc: large alloc 3363667968 bytes == 0x557c6dc3a000 @  0x7fbfd2f791e7 0x557b7f9e8772 0x557b7f9d27aa 0x557b7f9d31c8 0x557b7f961cb1 0x557b7f959c2e 0x557b7f95c2e9 0x557b7f947096 0x7fbfd1112b97 0x557b7f948ada\n",
            "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
            "tcmalloc: large alloc 3786956800 bytes == 0x557e6206e000 @  0x7fbfd2f791e7 0x557b7f9e8772 0x557b7f9d27aa 0x557b7f9d31c8 0x557b7f961cb1 0x557b7f959c2e 0x557b7f95c2e9 0x557b7f947096 0x7fbfd1112b97 0x557b7f948ada\n",
            "####################################################################################################\n",
            "=== 4/5 Calculating and writing order-interpolated probabilities ===\n",
            "Chain sizes: 1:10571820 2:251547888 3:1203595120 4:2142721024 5:3124801536 6:3786955264\n",
            "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
            "tcmalloc: large alloc 3786956800 bytes == 0x557c6dc36000 @  0x7fbfd2f791e7 0x557b7f9e8772 0x557b7f9d27aa 0x557b7f9d31c8 0x557b7f95c6f1 0x557b7f947096 0x7fbfd1112b97 0x557b7f948ada\n",
            "####################################################################################################\n",
            "=== 5/5 Writing ARPA model ===\n",
            "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n",
            "****************************************************************************************************\n",
            "Name:lmplz\tVmPeak:17714608 kB\tVmRSS:10303952 kB\tRSSMax:10647124 kB\tuser:741.912\tsys:137.195\tCPU:879.107\treal:2079.64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oeq7b-Zc95yq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 958
        },
        "outputId": "e1fa1668-e874-478e-b0ea-35dffbabd6e8"
      },
      "source": [
        "ls"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " \u001b[0m\u001b[01;34mbin\u001b[0m/                                      nyt_eng_199801   nyt_eng_200209\n",
            " CMakeCache.txt                            nyt_eng_199802   nyt_eng_200210\n",
            " \u001b[01;34mCMakeFiles\u001b[0m/                               nyt_eng_199803   nyt_eng_200211\n",
            " cmake_install.cmake                       nyt_eng_199804   nyt_eng_200212\n",
            "'decompose_n_add_token[i]_in_J.txt'        nyt_eng_199805   nyt_eng_200301\n",
            " file_for_evaluate.txt                     nyt_eng_199806   nyt_eng_200302\n",
            " gec-test-set.txt                          nyt_eng_199807   nyt_eng_200303\n",
            " gigaword3_nyt_eng.tar.gz                  nyt_eng_199808   nyt_eng_200304\n",
            " \u001b[01;34mGutenberg\u001b[0m/                                nyt_eng_199809   nyt_eng_200305\n",
            " Gutenberg.zip                             nyt_eng_199810   nyt_eng_200306\n",
            " \u001b[01;34mlib\u001b[0m/                                      nyt_eng_199811   nyt_eng_200307\n",
            " \u001b[01;34mlm\u001b[0m/                                       nyt_eng_199812   nyt_eng_200308\n",
            " Makefile                                  nyt_eng_199901   nyt_eng_200309\n",
            "'notdecompose_but_add_token[i]_in_J.txt'   nyt_eng_199902   nyt_eng_200310\n",
            " nyt_eng_199407                            nyt_eng_199903   nyt_eng_200311\n",
            " nyt_eng_199408                            nyt_eng_199904   nyt_eng_200312\n",
            " nyt_eng_199409                            nyt_eng_199905   nyt_eng_200401\n",
            " nyt_eng_199410                            nyt_eng_199906   nyt_eng_200402\n",
            " nyt_eng_199411                            nyt_eng_199907   nyt_eng_200403\n",
            " nyt_eng_199412                            nyt_eng_199908   nyt_eng_200404\n",
            " nyt_eng_199501                            nyt_eng_199909   nyt_eng_200405\n",
            " nyt_eng_199502                            nyt_eng_199910   nyt_eng_200407\n",
            " nyt_eng_199503                            nyt_eng_199911   nyt_eng_200408\n",
            " nyt_eng_199504                            nyt_eng_199912   nyt_eng_200409\n",
            " nyt_eng_199505                            nyt_eng_200001   nyt_eng_200410\n",
            " nyt_eng_199506                            nyt_eng_200002   nyt_eng_200411\n",
            " nyt_eng_199507                            nyt_eng_200003   nyt_eng_200412\n",
            " nyt_eng_199508                            nyt_eng_200004   nyt_eng_200501\n",
            " nyt_eng_199509                            nyt_eng_200005   nyt_eng_200502\n",
            " nyt_eng_199510                            nyt_eng_200006   nyt_eng_200503\n",
            " nyt_eng_199511                            nyt_eng_200007   nyt_eng_200504\n",
            " nyt_eng_199512                            nyt_eng_200008   nyt_eng_200505\n",
            " nyt_eng_199601                            nyt_eng_200009   nyt_eng_200506\n",
            " nyt_eng_199602                            nyt_eng_200010   nyt_eng_200507\n",
            " nyt_eng_199603                            nyt_eng_200011   nyt_eng_200508\n",
            " nyt_eng_199604                            nyt_eng_200012   nyt_eng_200509\n",
            " nyt_eng_199605                            nyt_eng_200101   nyt_eng_200510\n",
            " nyt_eng_199606                            nyt_eng_200102   nyt_eng_200511\n",
            " nyt_eng_199607                            nyt_eng_200103   nyt_eng_200512\n",
            " nyt_eng_199608                            nyt_eng_200104   nyt_eng_200601\n",
            " nyt_eng_199609                            nyt_eng_200105   nyt_eng_200602\n",
            " nyt_eng_199610                            nyt_eng_200106   nyt_eng_200603\n",
            " nyt_eng_199611                            nyt_eng_200107   nyt_eng_200604\n",
            " nyt_eng_199612                            nyt_eng_200108   nyt_eng_200605\n",
            " nyt_eng_199701                            nyt_eng_200109   nyt_eng_200606\n",
            " nyt_eng_199702                            nyt_eng_200110   nyt_eng_200607\n",
            " nyt_eng_199703                            nyt_eng_200111   nyt_eng_200608\n",
            " nyt_eng_199704                            nyt_eng_200112   nyt_eng_200609\n",
            " nyt_eng_199705                            nyt_eng_200201   nyt_eng_200610\n",
            " nyt_eng_199706                            nyt_eng_200202   nyt_eng_200611\n",
            " nyt_eng_199707                            nyt_eng_200203   nyt_eng_200612\n",
            " nyt_eng_199708                            nyt_eng_200204   nyt_eng.txt\n",
            " nyt_eng_199709                            nyt_eng_200205   sixgram.arpa\n",
            " nyt_eng_199710                            nyt_eng_200206   \u001b[01;34mutil\u001b[0m/\n",
            " nyt_eng_199711                            nyt_eng_200207\n",
            " nyt_eng_199712                            nyt_eng_200208\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tVX6Rroz0amG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import kenlm\n",
        "model = kenlm.Model('sixgram.arpa')"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8KTUQHvt-DuN",
        "colab_type": "text"
      },
      "source": [
        "#Evaluate Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lhKkZCqQ-Gx1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate(model, s):\n",
        "  tokens = s.split(' ')\n",
        "  log_score = 0.0\n",
        "  for i, (logprob, length, oov) in enumerate(model.full_scores(s,bos = True, eos = True)):\n",
        "    if i < len(tokens):\n",
        "     tokens[i], math.exp(logprob), oov\n",
        "    else:\n",
        "      'END', math.exp(logprob), oov\n",
        "  \n",
        "    log_score += logprob\n",
        "  return log_score"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JVkv4KZO-KFX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import math\n",
        "def print_score(model, s):\n",
        "  tokens = s.split(' ')\n",
        "  log_score = 0.0\n",
        "  for i, (logprob, length, oov) in enumerate(model.full_scores(s)):\n",
        "    if i < len(tokens):\n",
        "      print(tokens[i], math.exp(logprob), oov)\n",
        "    else:\n",
        "      print('END', math.exp(logprob), oov)\n",
        "  \n",
        "    log_score += logprob\n",
        "  return log_score"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QaHNzA9m-kvZ",
        "colab_type": "text"
      },
      "source": [
        "#Edit function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EcAE5qIp-ryU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def edit_noun(word,tokens,i):\n",
        "      sentence_candidates = []\n",
        "      default = word\n",
        "      stem = lemmatizer.lemmatize(word.lower())\n",
        "      for candidate in noun_candidate(stem):\n",
        "          new_tokens = tokens[:]\n",
        "          new_tokens[i] = candidate\n",
        "          new_sentence = \" \".join(new_tokens)\n",
        "          sentence_candidates.append((candidate,evaluate(model,new_sentence)))\n",
        "      rank = sorted(sentence_candidates , key = lambda x:x[1],reverse=True)\n",
        "      best_candidate = rank[0][0]\n",
        "      if default[0].isupper() == True:\n",
        "        best_candidate = best_candidate.capitalize()\n",
        "      return best_candidate"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F93UeawD-sbZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " def edit_art(word,tokens,i): \n",
        "      sentence_candidates = []\n",
        "      for candidate in article_candidate():\n",
        "          new_tokens = tokens[:]\n",
        "          new_tokens[i] = candidate\n",
        "          new_sentence = \" \".join(new_tokens)\n",
        "          sentence_candidates.append((candidate,evaluate(model,new_sentence)))\n",
        "      rank = sorted(sentence_candidates , key = lambda x:x[1],reverse=True)\n",
        "      best_candidate = rank[0][0]\n",
        "      return best_candidate"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uPihT6oA-vWx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def edit_be_verb(word,tokens,i):    \n",
        "      sentence_candidates = []\n",
        "      for candidate in be_verb():\n",
        "          new_tokens = tokens[:]\n",
        "          new_tokens[i] = candidate\n",
        "          new_sentence = \" \".join(new_tokens)\n",
        "          sentence_candidates.append((candidate,evaluate(model,new_sentence)))\n",
        "      rank = sorted(sentence_candidates , key = lambda x:x[1],reverse=True)\n",
        "      best_candidate = rank[0][0]\n",
        "      return best_candidate"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LtIhX4Zu-xuF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def edit_verb(word,tokens,i):\n",
        "      sentence_candidates = []\n",
        "      default = word\n",
        "      stem = nltk.stem.WordNetLemmatizer().lemmatize(word, 'v')\n",
        "\n",
        "      for candidate in verb_form_candidate(stem):\n",
        "        new_tokens = tokens[:]\n",
        "        new_tokens[i] = candidate\n",
        "        new_sentence = \" \".join(new_tokens)\n",
        "        sentence_candidates.append((candidate,evaluate(model,new_sentence)))\n",
        "      rank = sorted(sentence_candidates , key = lambda x:x[1],reverse=True)\n",
        "      best_candidate = rank[0][0]\n",
        "      if default[0].isupper() == True:\n",
        "        best_candidate = best_candidate.capitalize()\n",
        "      return best_candidate"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fd5mbxLK-3Oe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def edit_passive_voice(word,tokens,i):\n",
        "      sentence_candidates = []\n",
        "      stem = lemmatizer.lemmatize(word.lower())\n",
        "      if stem in verb_form_candidate(stem):\n",
        "        for vj_can in verb_form_candidate(stem):        \n",
        "          new_tokens = tokens[:]\n",
        "          new_tokens[i] = vj_can\n",
        "          sentence = \" \".join(new_tokens)\n",
        "          sentence_candidates.append((vj_can,evaluate(model,sentence)))\n",
        "        rank = sorted(sentence_candidates , key = lambda x:x[1],reverse=True)\n",
        "        best_candidate = rank[0][0]\n",
        "      else:\n",
        "        best_candidate = word\n",
        "      return best_candidate"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eTc3sCf2-6Ku",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def edit_prep(word,tokens,i):   \n",
        "      sentence_candidates = []\n",
        "      for candidate in prep_candidate():\n",
        "        new_tokens = tokens[:]\n",
        "        new_tokens[i] = candidate\n",
        "        new_sentence = \" \".join(new_tokens)\n",
        "        sentence_candidates.append((candidate,evaluate(model,new_sentence)))\n",
        "      rank = sorted(sentence_candidates , key = lambda x:x[1],reverse=True)\n",
        "      best_candidate = rank[0][0]\n",
        "      return best_candidate"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gbc0WQDmS6Il",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def join_tokens(tokens):\n",
        "  result = \" \".join(tokens)\n",
        "  if len(result) > 2:\n",
        "    result = result[0].upper() + result[1:]\n",
        "  else:\n",
        "    result = result\n",
        "  return result"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QMvtwD6r-_SO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def edit_sentence7(sentence):\n",
        "  pos_sentence = pos_tag(sentence)\n",
        "  urls = re.findall(r\"https?\", sentence)\n",
        "  if len(urls) > 0:\n",
        "    return sentence\n",
        "  else:\n",
        "   tokens = [token for token,pos in pos_sentence]\n",
        "   for i in range(len(pos_sentence)): \n",
        "  \n",
        "  #Noun    \n",
        "    if pos_sentence[i][1].startswith(\"N\"):\n",
        "      tokens[i] = edit_noun(pos_sentence[i][0],tokens,i)\n",
        "\n",
        "    #Article\n",
        "    elif pos_sentence[i][0] in article_candidate():\n",
        "      tokens[i] = edit_art(pos_sentence[i][0],tokens,i)\n",
        "\n",
        "    #Be_verb\n",
        "    elif pos_sentence[i][0] in be_verb():\n",
        "      tokens[i] = edit_be_verb(pos_sentence[i][0],tokens,i)\n",
        "\n",
        "    #Verb_forms\n",
        "    elif pos_sentence[i][1].startswith(\"V\"):\n",
        "      tokens[i] = edit_verb(pos_sentence[i][0],tokens,i)\n",
        "\n",
        "    elif pos_sentence[i][1].startswith(\"J\"):\n",
        "      tokens[i] = edit_passive_voice(pos_sentence[i][0],tokens,i)\n",
        "\n",
        "    #Preposition\n",
        "    elif pos_sentence[i][0] in prep_candidate():\n",
        "      tokens[i] = edit_prep(pos_sentence[i][0],tokens,i)\n",
        "  return join_tokens(tokens)"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OuKDZcivTQ2T",
        "colab_type": "text"
      },
      "source": [
        "#Evluate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tR479F9dTUi9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "da74a753-b17d-40e8-9ae3-ac2fdf4457aa"
      },
      "source": [
        "with open('/content/contest2/bea19-sentences.txt') as file:\n",
        "  data = file.read().split('\\n')\n",
        "  data = [s for s in data]\n",
        "data[100]"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"According Khandani 's article , engineers can test the solution by prototyping or concurrent engineering .\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EqCwfUubTYMy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 116,
          "referenced_widgets": [
            "bbc4474c843543ceb43dd335f3aecfb0",
            "7a3ded7900d54fb0ac6dfad5a18e752e",
            "c1dab40291c84d368ade0236d95f4cef",
            "d170d5ebd4d4405a96ac85c23a1e4d6d",
            "3f7ff21318674b5b81cf84e2aed6793b",
            "2aec8c1fc74b4200b336a93367dc5b18",
            "5d9c80f8dbb74aabaf8c6a01d20c8ef6",
            "65d18e8ff1744a19b522379d283d16fb"
          ]
        },
        "outputId": "3cebe8d1-2b12-45ba-8727-88438648d7e0"
      },
      "source": [
        "from tqdm import tqdm_notebook\n",
        "with open('file_for_evaluate.txt', 'w') as f :\n",
        "  for i in tqdm_notebook(range(len(data))):\n",
        "    if i+1 < len(data):\n",
        "      f.write(edit_sentence7(data[i]) + '\\n')\n",
        "    else:\n",
        "      f.write(edit_sentence7(data[i]))"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bbc4474c843543ceb43dd335f3aecfb0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=57152.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kzx_hw0TLk6d",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "7e7c9e9d-4de4-4976-d942-604ee9df4be0"
      },
      "source": [
        "!python /content/contest2/m2scorer/scripts/m2scorer.py /content/kenlm/build/file_for_evaluate.txt /content/contest2/nucle.train.gold.bea19.m2"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Precision   : 0.0680\n",
            "Recall      : 0.1767\n",
            "F_0.5       : 0.0775\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lFNiIVz3Tc22",
        "colab_type": "text"
      },
      "source": [
        "#Generate Test file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KgDBGWLOUyEF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "6bf55d28-8872-46c1-e0cd-e6be5b038932"
      },
      "source": [
        "!gdown --id 1CHnRNybDYbq9xTZNCxf22PIQIyBe0Yfz"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1CHnRNybDYbq9xTZNCxf22PIQIyBe0Yfz\n",
            "To: /content/kenlm/build/gec-test-set.txt\n",
            "\r  0% 0.00/162k [00:00<?, ?B/s]\r100% 162k/162k [00:00<00:00, 69.4MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R5iEE62tU125",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "1d0ad784-9353-4770-abcf-c52ad00e32d7"
      },
      "source": [
        "with open('/content/kenlm/build/gec-test-set.txt') as file:\n",
        "  data = file.read().split('\\n')\n",
        "  sentences = [s for s in data]\n",
        "sentences[100]"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"Genetic testing is made possible and available for one 's decision to undergo .\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c9nUr_ADTf4Z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 116,
          "referenced_widgets": [
            "8c26d2ee6f4242f4a9f48aaf45b25acf",
            "70af676e709d4943b4726b852d671f90",
            "488c7adc69c8457c84f0bc0fd57ecb9a",
            "b0fb8633acd241b1a30f5d8c196e900d",
            "bc70964b308d4122ac9fe7a71344cfb4",
            "7cdff92d395147ad8789d6dc025c08c8",
            "c8f8627c971a4b72b82d98baf339d9cd",
            "7485f08f3b49470796a0cbe05a53e8f2"
          ]
        },
        "outputId": "f421e21e-0fad-4f93-cfe9-39f7c9d42d0c"
      },
      "source": [
        "from tqdm import tqdm_notebook\n",
        "with open('corrected_sentences.txt', 'w') as f :\n",
        "  for i in tqdm_notebook(range(len(sentences))):\n",
        "    if i+1 < len(data):\n",
        "      f.write(edit_sentence7(sentences[i]) + '\\n')\n",
        "    else:\n",
        "      f.write(edit_sentence7(sentences[i]))"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8c26d2ee6f4242f4a9f48aaf45b25acf",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1313.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}