Edureka_Sequence_Learning_Test.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Edureka Sequence Learning - Test",
      "provenance": [],
      "mount_file_id": "1BE36agLtXYX7CI-zWsTyiT3wviuS6X5j",
      "authorship_tag": "ABX9TyNmu+iRU1OuUG9fJVD1RsXq",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/nandinib1999/edureka-assignments/blob/main/Edureka_Sequence_Learning_Test.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "50LKFYTcdQNP"
      },
      "source": [
        "### **DESCRIPTION**\n",
        "\n",
        "Predict named entity from given data and showcase how entities like persons, locations, organizations, and \n",
        "other miscellaneous entity names of that do not belong to the previous three groups can be predicted from \n",
        "the tagged dataset. \n",
        "\n",
        "### **Problem Statement**\n",
        "\n",
        "Using CoNLL 2003 data, predict tagged NER using CRF Algorithm. Also, tune the algorithm and explore the \n",
        "learnings that have been done by the CRF Model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XC9_XPjSSbuB"
      },
      "source": [
        "# Reading the input files for training and evaluation of CRF model\n",
        "with open(\"/content/drive/MyDrive/sl_train.txt\") as fp:\n",
        "  train_data = fp.readlines()\n",
        "\n",
        "with open(\"/content/drive/MyDrive/sl_test.txt\") as fp:\n",
        "  test_data = fp.readlines()"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N-spIRGSM0IM",
        "outputId": "789cc956-b427-4619-9f35-b955ec9e0c47"
      },
      "source": [
        "test_data=test_data[2:]\n",
        "test_data[:20]"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['SOCCER NN B-NP O\\n',\n",
              " '- : O O\\n',\n",
              " 'JAPAN NNP B-NP B-LOC\\n',\n",
              " 'GET VB B-VP O\\n',\n",
              " 'LUCKY NNP B-NP O\\n',\n",
              " 'WIN NNP I-NP O\\n',\n",
              " ', , O O\\n',\n",
              " 'CHINA NNP B-NP B-PER\\n',\n",
              " 'IN IN B-PP O\\n',\n",
              " 'SURPRISE DT B-NP O\\n',\n",
              " 'DEFEAT NN I-NP O\\n',\n",
              " '. . O O\\n',\n",
              " '\\n',\n",
              " 'Nadim NNP B-NP B-PER\\n',\n",
              " 'Ladki NNP I-NP I-PER\\n',\n",
              " '\\n',\n",
              " 'AL-AIN NNP B-NP B-LOC\\n',\n",
              " ', , O O\\n',\n",
              " 'United NNP B-NP B-LOC\\n',\n",
              " 'Arab NNP I-NP I-LOC\\n']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zPyhdAVhMw7r",
        "outputId": "e0146104-5f3a-40bc-84a8-b5033b17c2c0"
      },
      "source": [
        "train_data=train_data[2:]\n",
        "train_data[:20]"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['EU NNP B-NP B-ORG\\n',\n",
              " 'rejects VBZ B-VP O\\n',\n",
              " 'German JJ B-NP B-MISC\\n',\n",
              " 'call NN I-NP O\\n',\n",
              " 'to TO B-VP O\\n',\n",
              " 'boycott VB I-VP O\\n',\n",
              " 'British JJ B-NP B-MISC\\n',\n",
              " 'lamb NN I-NP O\\n',\n",
              " '. . O O\\n',\n",
              " '\\n',\n",
              " 'Peter NNP B-NP B-PER\\n',\n",
              " 'Blackburn NNP I-NP I-PER\\n',\n",
              " '\\n',\n",
              " 'BRUSSELS NNP B-NP B-LOC\\n',\n",
              " '1996-08-22 CD I-NP O\\n',\n",
              " '\\n',\n",
              " 'The DT B-NP O\\n',\n",
              " 'European NNP I-NP B-ORG\\n',\n",
              " 'Commission NNP I-NP I-ORG\\n',\n",
              " 'said VBD B-VP O\\n']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z79bUsV7NNdm"
      },
      "source": [
        "# Splitting the text from file into sentences based on newline character\n",
        "def split_into_sentences(data_lst):\n",
        "  final_lst = []\n",
        "  tmp = []\n",
        "  for token in data_lst:\n",
        "    if token == \"\\n\":\n",
        "      final_lst.append(tmp)\n",
        "      tmp = []\n",
        "    else:\n",
        "      tmp.append(tuple(token.strip().split(' ')))\n",
        "  if len(tmp) > 0:\n",
        "    final_lst.append(tmp)\n",
        "  return final_lst"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fc_t1dvsS6Cn",
        "outputId": "7f66dcfc-92d7-426c-884c-a0fb79633a4d"
      },
      "source": [
        "train_sentences = split_into_sentences(train_data)\n",
        "train_sentences[:5]"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[('EU', 'NNP', 'B-NP', 'B-ORG'),\n",
              "  ('rejects', 'VBZ', 'B-VP', 'O'),\n",
              "  ('German', 'JJ', 'B-NP', 'B-MISC'),\n",
              "  ('call', 'NN', 'I-NP', 'O'),\n",
              "  ('to', 'TO', 'B-VP', 'O'),\n",
              "  ('boycott', 'VB', 'I-VP', 'O'),\n",
              "  ('British', 'JJ', 'B-NP', 'B-MISC'),\n",
              "  ('lamb', 'NN', 'I-NP', 'O'),\n",
              "  ('.', '.', 'O', 'O')],\n",
              " [('Peter', 'NNP', 'B-NP', 'B-PER'), ('Blackburn', 'NNP', 'I-NP', 'I-PER')],\n",
              " [('BRUSSELS', 'NNP', 'B-NP', 'B-LOC'), ('1996-08-22', 'CD', 'I-NP', 'O')],\n",
              " [('The', 'DT', 'B-NP', 'O'),\n",
              "  ('European', 'NNP', 'I-NP', 'B-ORG'),\n",
              "  ('Commission', 'NNP', 'I-NP', 'I-ORG'),\n",
              "  ('said', 'VBD', 'B-VP', 'O'),\n",
              "  ('on', 'IN', 'B-PP', 'O'),\n",
              "  ('Thursday', 'NNP', 'B-NP', 'O'),\n",
              "  ('it', 'PRP', 'B-NP', 'O'),\n",
              "  ('disagreed', 'VBD', 'B-VP', 'O'),\n",
              "  ('with', 'IN', 'B-PP', 'O'),\n",
              "  ('German', 'JJ', 'B-NP', 'B-MISC'),\n",
              "  ('advice', 'NN', 'I-NP', 'O'),\n",
              "  ('to', 'TO', 'B-PP', 'O'),\n",
              "  ('consumers', 'NNS', 'B-NP', 'O'),\n",
              "  ('to', 'TO', 'B-VP', 'O'),\n",
              "  ('shun', 'VB', 'I-VP', 'O'),\n",
              "  ('British', 'JJ', 'B-NP', 'B-MISC'),\n",
              "  ('lamb', 'NN', 'I-NP', 'O'),\n",
              "  ('until', 'IN', 'B-SBAR', 'O'),\n",
              "  ('scientists', 'NNS', 'B-NP', 'O'),\n",
              "  ('determine', 'VBP', 'B-VP', 'O'),\n",
              "  ('whether', 'IN', 'B-SBAR', 'O'),\n",
              "  ('mad', 'JJ', 'B-NP', 'O'),\n",
              "  ('cow', 'NN', 'I-NP', 'O'),\n",
              "  ('disease', 'NN', 'I-NP', 'O'),\n",
              "  ('can', 'MD', 'B-VP', 'O'),\n",
              "  ('be', 'VB', 'I-VP', 'O'),\n",
              "  ('transmitted', 'VBN', 'I-VP', 'O'),\n",
              "  ('to', 'TO', 'B-PP', 'O'),\n",
              "  ('sheep', 'NN', 'B-NP', 'O'),\n",
              "  ('.', '.', 'O', 'O')],\n",
              " [('Germany', 'NNP', 'B-NP', 'B-LOC'),\n",
              "  (\"'s\", 'POS', 'B-NP', 'O'),\n",
              "  ('representative', 'NN', 'I-NP', 'O'),\n",
              "  ('to', 'TO', 'B-PP', 'O'),\n",
              "  ('the', 'DT', 'B-NP', 'O'),\n",
              "  ('European', 'NNP', 'I-NP', 'B-ORG'),\n",
              "  ('Union', 'NNP', 'I-NP', 'I-ORG'),\n",
              "  (\"'s\", 'POS', 'B-NP', 'O'),\n",
              "  ('veterinary', 'JJ', 'I-NP', 'O'),\n",
              "  ('committee', 'NN', 'I-NP', 'O'),\n",
              "  ('Werner', 'NNP', 'I-NP', 'B-PER'),\n",
              "  ('Zwingmann', 'NNP', 'I-NP', 'I-PER'),\n",
              "  ('said', 'VBD', 'B-VP', 'O'),\n",
              "  ('on', 'IN', 'B-PP', 'O'),\n",
              "  ('Wednesday', 'NNP', 'B-NP', 'O'),\n",
              "  ('consumers', 'NNS', 'I-NP', 'O'),\n",
              "  ('should', 'MD', 'B-VP', 'O'),\n",
              "  ('buy', 'VB', 'I-VP', 'O'),\n",
              "  ('sheepmeat', 'NN', 'B-NP', 'O'),\n",
              "  ('from', 'IN', 'B-PP', 'O'),\n",
              "  ('countries', 'NNS', 'B-NP', 'O'),\n",
              "  ('other', 'JJ', 'B-ADJP', 'O'),\n",
              "  ('than', 'IN', 'B-PP', 'O'),\n",
              "  ('Britain', 'NNP', 'B-NP', 'B-LOC'),\n",
              "  ('until', 'IN', 'B-SBAR', 'O'),\n",
              "  ('the', 'DT', 'B-NP', 'O'),\n",
              "  ('scientific', 'JJ', 'I-NP', 'O'),\n",
              "  ('advice', 'NN', 'I-NP', 'O'),\n",
              "  ('was', 'VBD', 'B-VP', 'O'),\n",
              "  ('clearer', 'JJR', 'B-ADJP', 'O'),\n",
              "  ('.', '.', 'O', 'O')]]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sLbyYAJw42NN",
        "outputId": "134af8fe-d733-429a-defc-f69ff4ea4022"
      },
      "source": [
        "test_sentences = split_into_sentences(test_data)\n",
        "test_sentences[:5]"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[('SOCCER', 'NN', 'B-NP', 'O'),\n",
              "  ('-', ':', 'O', 'O'),\n",
              "  ('JAPAN', 'NNP', 'B-NP', 'B-LOC'),\n",
              "  ('GET', 'VB', 'B-VP', 'O'),\n",
              "  ('LUCKY', 'NNP', 'B-NP', 'O'),\n",
              "  ('WIN', 'NNP', 'I-NP', 'O'),\n",
              "  (',', ',', 'O', 'O'),\n",
              "  ('CHINA', 'NNP', 'B-NP', 'B-PER'),\n",
              "  ('IN', 'IN', 'B-PP', 'O'),\n",
              "  ('SURPRISE', 'DT', 'B-NP', 'O'),\n",
              "  ('DEFEAT', 'NN', 'I-NP', 'O'),\n",
              "  ('.', '.', 'O', 'O')],\n",
              " [('Nadim', 'NNP', 'B-NP', 'B-PER'), ('Ladki', 'NNP', 'I-NP', 'I-PER')],\n",
              " [('AL-AIN', 'NNP', 'B-NP', 'B-LOC'),\n",
              "  (',', ',', 'O', 'O'),\n",
              "  ('United', 'NNP', 'B-NP', 'B-LOC'),\n",
              "  ('Arab', 'NNP', 'I-NP', 'I-LOC'),\n",
              "  ('Emirates', 'NNPS', 'I-NP', 'I-LOC'),\n",
              "  ('1996-12-06', 'CD', 'I-NP', 'O')],\n",
              " [('Japan', 'NNP', 'B-NP', 'B-LOC'),\n",
              "  ('began', 'VBD', 'B-VP', 'O'),\n",
              "  ('the', 'DT', 'B-NP', 'O'),\n",
              "  ('defence', 'NN', 'I-NP', 'O'),\n",
              "  ('of', 'IN', 'B-PP', 'O'),\n",
              "  ('their', 'PRP$', 'B-NP', 'O'),\n",
              "  ('Asian', 'JJ', 'I-NP', 'B-MISC'),\n",
              "  ('Cup', 'NNP', 'I-NP', 'I-MISC'),\n",
              "  ('title', 'NN', 'I-NP', 'O'),\n",
              "  ('with', 'IN', 'B-PP', 'O'),\n",
              "  ('a', 'DT', 'B-NP', 'O'),\n",
              "  ('lucky', 'JJ', 'I-NP', 'O'),\n",
              "  ('2-1', 'CD', 'I-NP', 'O'),\n",
              "  ('win', 'VBP', 'B-VP', 'O'),\n",
              "  ('against', 'IN', 'B-PP', 'O'),\n",
              "  ('Syria', 'NNP', 'B-NP', 'B-LOC'),\n",
              "  ('in', 'IN', 'B-PP', 'O'),\n",
              "  ('a', 'DT', 'B-NP', 'O'),\n",
              "  ('Group', 'NNP', 'I-NP', 'O'),\n",
              "  ('C', 'NNP', 'I-NP', 'O'),\n",
              "  ('championship', 'NN', 'I-NP', 'O'),\n",
              "  ('match', 'NN', 'I-NP', 'O'),\n",
              "  ('on', 'IN', 'B-PP', 'O'),\n",
              "  ('Friday', 'NNP', 'B-NP', 'O'),\n",
              "  ('.', '.', 'O', 'O')],\n",
              " [('But', 'CC', 'O', 'O'),\n",
              "  ('China', 'NNP', 'B-NP', 'B-LOC'),\n",
              "  ('saw', 'VBD', 'B-VP', 'O'),\n",
              "  ('their', 'PRP$', 'B-NP', 'O'),\n",
              "  ('luck', 'NN', 'I-NP', 'O'),\n",
              "  ('desert', 'VB', 'B-VP', 'O'),\n",
              "  ('them', 'PRP', 'B-NP', 'O'),\n",
              "  ('in', 'IN', 'B-PP', 'O'),\n",
              "  ('the', 'DT', 'B-NP', 'O'),\n",
              "  ('second', 'NN', 'I-NP', 'O'),\n",
              "  ('match', 'NN', 'I-NP', 'O'),\n",
              "  ('of', 'IN', 'B-PP', 'O'),\n",
              "  ('the', 'DT', 'B-NP', 'O'),\n",
              "  ('group', 'NN', 'I-NP', 'O'),\n",
              "  (',', ',', 'O', 'O'),\n",
              "  ('crashing', 'VBG', 'B-VP', 'O'),\n",
              "  ('to', 'TO', 'B-PP', 'O'),\n",
              "  ('a', 'DT', 'B-NP', 'O'),\n",
              "  ('surprise', 'NN', 'I-NP', 'O'),\n",
              "  ('2-0', 'CD', 'I-NP', 'O'),\n",
              "  ('defeat', 'NN', 'I-NP', 'O'),\n",
              "  ('to', 'TO', 'B-PP', 'O'),\n",
              "  ('newcomers', 'NNS', 'B-NP', 'O'),\n",
              "  ('Uzbekistan', 'NNP', 'I-NP', 'B-LOC'),\n",
              "  ('.', '.', 'O', 'O')]]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rL01RqS8B1xi"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "def sent2df(sent):\n",
        "  df = pd.DataFrame(data=sent, columns=['word', 'pos', 'parse', 'ner'])\n",
        "  df.index.name = 'seq_num'\n",
        "  return df\n",
        "    \n",
        "def all_sentences(sents):\n",
        "  total_df = [sent2df(s) for s in sents]\n",
        "  return total_df"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dltwkPEqB5_h"
      },
      "source": [
        "train_sents = all_sentences(train_sentences)\n",
        "test_sents  = all_sentences(test_sentences)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "id": "e9g-JOaXCFc5",
        "outputId": "7d7c48b0-3274-49a1-ee87-db5dfe859f48"
      },
      "source": [
        "train_sents[0]"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>pos</th>\n",
              "      <th>parse</th>\n",
              "      <th>ner</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>word_seq_num</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>EU</td>\n",
              "      <td>NNP</td>\n",
              "      <td>B-NP</td>\n",
              "      <td>B-ORG</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>rejects</td>\n",
              "      <td>VBZ</td>\n",
              "      <td>B-VP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>German</td>\n",
              "      <td>JJ</td>\n",
              "      <td>B-NP</td>\n",
              "      <td>B-MISC</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>call</td>\n",
              "      <td>NN</td>\n",
              "      <td>I-NP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>to</td>\n",
              "      <td>TO</td>\n",
              "      <td>B-VP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>boycott</td>\n",
              "      <td>VB</td>\n",
              "      <td>I-VP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>British</td>\n",
              "      <td>JJ</td>\n",
              "      <td>B-NP</td>\n",
              "      <td>B-MISC</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>lamb</td>\n",
              "      <td>NN</td>\n",
              "      <td>I-NP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>.</td>\n",
              "      <td>.</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 word  pos parse     ner\n",
              "word_seq_num                            \n",
              "0                  EU  NNP  B-NP   B-ORG\n",
              "1             rejects  VBZ  B-VP       O\n",
              "2              German   JJ  B-NP  B-MISC\n",
              "3                call   NN  I-NP       O\n",
              "4                  to   TO  B-VP       O\n",
              "5             boycott   VB  I-VP       O\n",
              "6             British   JJ  B-NP  B-MISC\n",
              "7                lamb   NN  I-NP       O\n",
              "8                   .    .     O       O"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LHsDqjZh_x4Z"
      },
      "source": [
        "def get_labels(all_sents):\n",
        "    all_labels = []\n",
        "    \n",
        "    for sent_df in all_sents:\n",
        "        labels = sent_df['ner'].tolist()\n",
        "        all_labels.append(labels)\n",
        "        \n",
        "    return all_labels \n",
        "\n",
        "\n",
        "def word2features(i, sent_df):    \n",
        "    word, postag = sent_df.iloc[i].loc[['word', 'pos']]\n",
        "    \n",
        "    features = {\n",
        "        'bias': 1.0,\n",
        "        'word.lower()': word.lower(),\n",
        "        'word[-3:]': word[-3:],\n",
        "        'word[-2:]': word[-2:],\n",
        "        'word.isupper()': word.isupper(),\n",
        "        'word.istitle()': word.istitle(),\n",
        "        'word.isdigit()': word.isdigit(),\n",
        "        'postag': postag,\n",
        "        'postag[:2]': postag[:2],\n",
        "    }\n",
        "    if i > 0:\n",
        "        word1, postag1 = sent_df.iloc[i-1].loc[['word', 'pos']]\n",
        "        \n",
        "        features.update({\n",
        "            '-1:word.lower()': word1.lower(),\n",
        "            '-1:word.istitle()': word1.istitle(),\n",
        "            '-1:word.isupper()': word1.isupper(),\n",
        "            '-1:postag': postag1,\n",
        "            '-1:postag[:2]': postag1[:2],\n",
        "        })\n",
        "    else:\n",
        "        features['BOS'] = True\n",
        "\n",
        "    if i < (sent_df.shape[0] - 1):\n",
        "        word1, postag1 = sent_df.iloc[i+1].loc[['word', 'pos']]\n",
        "        \n",
        "        features.update({\n",
        "            '+1:word.lower()': word1.lower(),\n",
        "            '+1:word.istitle()': word1.istitle(),\n",
        "            '+1:word.isupper()': word1.isupper(),\n",
        "            '+1:postag': postag1,\n",
        "            '+1:postag[:2]': postag1[:2],\n",
        "        })\n",
        "    else:\n",
        "        features['EOS'] = True\n",
        "\n",
        "    return features\n",
        "\n",
        "def sent2features(df):\n",
        "    features = df.index.map(lambda x: word2features(x, df))\n",
        "    return features.tolist()\n",
        "\n",
        "def get_feature_values(all_sents):\n",
        "    all_features = [sent2features(s) for s in all_sents]    \n",
        "    return all_features"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1F2hFVOECMXZ"
      },
      "source": [
        "X_train = get_feature_values(train_sents)\n",
        "y_train = get_labels(train_sents)\n",
        "\n",
        "X_test = get_feature_values(test_sents)\n",
        "y_test = get_labels(test_sents)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h4Td0rr748s6",
        "outputId": "278791ef-6cea-4997-86c7-841440d41c24"
      },
      "source": [
        "!pip install sklearn-crfsuite"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sklearn-crfsuite\n",
            "  Downloading https://files.pythonhosted.org/packages/25/74/5b7befa513482e6dee1f3dd68171a6c9dfc14c0eaa00f885ffeba54fe9b0/sklearn_crfsuite-0.3.6-py2.py3-none-any.whl\n",
            "Collecting python-crfsuite>=0.8.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/79/47/58f16c46506139f17de4630dbcfb877ce41a6355a1bbf3c443edb9708429/python_crfsuite-0.9.7-cp37-cp37m-manylinux1_x86_64.whl (743kB)\n",
            "\u001b[K     |████████████████████████████████| 747kB 6.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from sklearn-crfsuite) (0.8.9)\n",
            "Requirement already satisfied: tqdm>=2.0 in /usr/local/lib/python3.7/dist-packages (from sklearn-crfsuite) (4.41.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sklearn-crfsuite) (1.15.0)\n",
            "Installing collected packages: python-crfsuite, sklearn-crfsuite\n",
            "Successfully installed python-crfsuite-0.9.7 sklearn-crfsuite-0.3.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VGrineocfip7"
      },
      "source": [
        "### **Simple CRF Model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3NK1YTqM_rqY"
      },
      "source": [
        "import sklearn_crfsuite\n",
        "\n",
        "crf = sklearn_crfsuite.CRF(\n",
        "    algorithm='lbfgs',\n",
        "    c1=0.1,\n",
        "    c2=0.1,\n",
        "    max_iterations=100,\n",
        "    all_possible_transitions=True\n",
        ")"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zHVXHSZBDx1V",
        "outputId": "0d01c311-de33-4e8b-ae04-64ae6c22126c"
      },
      "source": [
        "%%time\n",
        "crf.fit(X_train, y_train)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 34.6 s, sys: 217 ms, total: 34.8 s\n",
            "Wall time: 34.7 s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/base.py:197: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
            "  FutureWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CRF(algorithm='lbfgs', all_possible_states=None, all_possible_transitions=True,\n",
              "    averaging=None, c=None, c1=0.1, c2=0.1, calibration_candidates=None,\n",
              "    calibration_eta=None, calibration_max_trials=None, calibration_rate=None,\n",
              "    calibration_samples=None, delta=None, epsilon=None, error_sensitive=None,\n",
              "    gamma=None, keep_tempfiles=None, linesearch=None, max_iterations=100,\n",
              "    max_linesearch=None, min_freq=None, model_filename=None, num_memories=None,\n",
              "    pa_type=None, period=None, trainer_cls=None, variance=None, verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pkxTkWOPEFn1",
        "outputId": "5b7d82f8-0412-4cdc-b10c-1edf5599d723"
      },
      "source": [
        "labels = list(crf.classes_)\n",
        "labels.remove('O')\n",
        "labels"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['B-ORG', 'B-MISC', 'B-PER', 'I-PER', 'B-LOC', 'I-ORG', 'I-MISC', 'I-LOC']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BPX9eE0efuvh"
      },
      "source": [
        "#### **Evaluation of model performance**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NJZRV8zOEKdb",
        "outputId": "9874423a-263f-46ce-93ba-299069fcf554"
      },
      "source": [
        "from sklearn_crfsuite import metrics\n",
        "\n",
        "y_pred = crf.predict(X_test)\n",
        "metrics.flat_f1_score(y_test, y_pred, average='weighted', labels=labels)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7994617872929973"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "es7wyJu-ESHa",
        "outputId": "f205ed9f-c5a3-4ffa-9ab8-9f4f946fb12d"
      },
      "source": [
        "print(metrics.flat_classification_report(y_test, y_pred, labels=labels, digits=3))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "       B-ORG      0.764     0.727     0.745      1661\n",
            "      B-MISC      0.816     0.756     0.785       702\n",
            "       B-PER      0.826     0.857     0.842      1617\n",
            "       I-PER      0.865     0.951     0.906      1156\n",
            "       B-LOC      0.853     0.808     0.830      1668\n",
            "       I-ORG      0.670     0.738     0.702       835\n",
            "      I-MISC      0.687     0.671     0.679       216\n",
            "       I-LOC      0.754     0.595     0.665       257\n",
            "\n",
            "   micro avg      0.801     0.800     0.800      8112\n",
            "   macro avg      0.779     0.763     0.769      8112\n",
            "weighted avg      0.802     0.800     0.799      8112\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g-m0kpVWgCee"
      },
      "source": [
        "#### **Hyperparameter Tuning**\n",
        "\n",
        "- GridSearchCV"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j9558BbbEzzl",
        "outputId": "c6bf4b34-0c64-4d54-8e83-2c981d8d9410"
      },
      "source": [
        "%%time\n",
        "\n",
        "import scipy\n",
        "from sklearn.metrics import make_scorer\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "\n",
        "# define fixed parameters and parameters to search\n",
        "crf = sklearn_crfsuite.CRF(\n",
        "    algorithm='lbfgs',\n",
        "    max_iterations=100,\n",
        "    all_possible_transitions=True\n",
        ")\n",
        "params_space = {\n",
        "    'c1': [0.5, 0.25, 0.125, 0.625],\n",
        "    'c2': [0.05, 0.025, 0.0125, 0.0625],\n",
        "}\n",
        "\n",
        "# use the same metric for evaluation\n",
        "f1_scorer = make_scorer(metrics.flat_f1_score, average='weighted', labels=labels)\n",
        "\n",
        "# search\n",
        "rs = GridSearchCV(crf, param_grid=params_space, cv=3, verbose=1, n_jobs=-1, scoring=f1_scorer)\n",
        "rs.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fitting 3 folds for each of 16 candidates, totalling 48 fits\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/base.py:197: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
            "  FutureWarning)\n",
            "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
            "[Parallel(n_jobs=-1)]: Done  48 out of  48 | elapsed: 26.8min finished\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 23min 47s, sys: 3.6 s, total: 23min 51s\n",
            "Wall time: 27min 19s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JTpSKYmDR1sW",
        "outputId": "287355c3-8d77-4543-9709-6d66a428849c"
      },
      "source": [
        "print('best params:', rs.best_params_)\n",
        "print('best CV score:', rs.best_score_)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "best params: {'c1': 0.125, 'c2': 0.0625}\n",
            "best CV score: 0.8430630416854501\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yvywkeCbR7eN",
        "outputId": "82b22b48-2450-4f5d-ae5f-2ff7f7e321b1"
      },
      "source": [
        "crf = rs.best_estimator_\n",
        "y_pred = crf.predict(X_test)\n",
        "print(metrics.flat_classification_report(\n",
        "    y_test, y_pred, labels=labels, digits=3\n",
        "))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "      B-MISC      0.819     0.759     0.788       702\n",
            "       B-PER      0.820     0.855     0.837      1617\n",
            "       I-PER      0.864     0.948     0.904      1156\n",
            "       B-LOC      0.853     0.813     0.832      1668\n",
            "       B-ORG      0.772     0.722     0.746      1661\n",
            "       I-ORG      0.680     0.734     0.706       835\n",
            "      I-MISC      0.670     0.667     0.668       216\n",
            "       I-LOC      0.746     0.607     0.670       257\n",
            "\n",
            "   micro avg      0.802     0.799     0.801      8112\n",
            "   macro avg      0.778     0.763     0.769      8112\n",
            "weighted avg      0.802     0.799     0.799      8112\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Ib-bSA0SNfR",
        "outputId": "4258033a-01f0-4043-f963-239a928e73e6"
      },
      "source": [
        "from collections import Counter\n",
        "\n",
        "def print_transitions(trans_features):\n",
        "    for (label_from, label_to), weight in trans_features:\n",
        "        print(\"%-6s -> %-7s %0.6f\" % (label_from, label_to, weight))\n",
        "\n",
        "print(\"Top likely transitions:\")\n",
        "print_transitions(Counter(crf.transition_features_).most_common(5))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Top likely transitions:\n",
            "I-MISC -> I-MISC  6.193969\n",
            "B-MISC -> I-MISC  5.909604\n",
            "B-LOC  -> I-LOC   5.390707\n",
            "I-LOC  -> I-LOC   4.975298\n",
            "B-PER  -> I-PER   4.664758\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xis-nm7jSSuY",
        "outputId": "c1cd8007-8e0c-493f-ba0d-008bdb5db86a"
      },
      "source": [
        "def print_state_features(state_features):\n",
        "    for (attr, label), weight in state_features:\n",
        "        print(\"%0.6f %-8s %s\" % (weight, label, attr))\n",
        "\n",
        "print(\"Top positive:\")\n",
        "print_state_features(Counter(crf.state_features_).most_common(5))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Top positive:\n",
            "6.726481 B-LOC    word.lower():chester-le-street\n",
            "6.300068 I-LOC    -1:word.lower():wisc\n",
            "6.295602 I-LOC    -1:word.lower():colo\n",
            "6.230998 B-PER    word.lower():clinton\n",
            "6.048541 B-LOC    +1:word.lower():1996-08-26\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LZz03T8cgXfT"
      },
      "source": [
        "- RandomizedSearchCV"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CHeR5EE-S0e6",
        "outputId": "093f467f-cadf-4117-adfb-ae8d1cb11ea5"
      },
      "source": [
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "from sklearn.metrics import make_scorer\n",
        "\n",
        "crf = sklearn_crfsuite.CRF(\n",
        "    algorithm='lbfgs',\n",
        "    max_iterations=100,\n",
        "    all_possible_transitions=True\n",
        ")\n",
        "\n",
        "params_space = {\n",
        "    'c1': [0.5, 0.25, 0.125, 0.625],\n",
        "    'c2': [0.05, 0.025, 0.0125, 0.0625],\n",
        "}\n",
        "\n",
        "# use the same metric for evaluation\n",
        "f1_scorer = make_scorer(metrics.flat_f1_score,\n",
        "                        average='weighted', labels=labels)\n",
        "\n",
        "# search\n",
        "rs = RandomizedSearchCV(crf, params_space,\n",
        "                        cv=3,\n",
        "                        verbose=1,\n",
        "                        n_jobs=-1,\n",
        "                        n_iter=50,\n",
        "                        scoring=f1_scorer)\n",
        "rs.fit(X_train, y_train)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fitting 3 folds for each of 16 candidates, totalling 48 fits\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/base.py:197: FutureWarning: From version 0.24, get_params will raise an AttributeError if a parameter cannot be retrieved as an instance attribute. Previously it would return None.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_search.py:281: UserWarning: The total space of parameters 16 is smaller than n_iter=50. Running 16 iterations. For exhaustive searches, use GridSearchCV.\n",
            "  % (grid_size, self.n_iter, grid_size), UserWarning)\n",
            "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
            "[Parallel(n_jobs=-1)]: Done  48 out of  48 | elapsed: 28.2min finished\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomizedSearchCV(cv=3, error_score=nan,\n",
              "                   estimator=CRF(algorithm='lbfgs', all_possible_states=None,\n",
              "                                 all_possible_transitions=True, averaging=None,\n",
              "                                 c=None, c1=None, c2=None,\n",
              "                                 calibration_candidates=None,\n",
              "                                 calibration_eta=None,\n",
              "                                 calibration_max_trials=None,\n",
              "                                 calibration_rate=None,\n",
              "                                 calibration_samples=None, delta=None,\n",
              "                                 epsilon=None, error_sensitive=None, gamma=None,\n",
              "                                 keep_...\n",
              "                                 trainer_cls=None, variance=None,\n",
              "                                 verbose=False),\n",
              "                   iid='deprecated', n_iter=50, n_jobs=-1,\n",
              "                   param_distributions={'c1': [0.5, 0.25, 0.125, 0.625],\n",
              "                                        'c2': [0.05, 0.025, 0.0125, 0.0625]},\n",
              "                   pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
              "                   return_train_score=False,\n",
              "                   scoring=make_scorer(flat_f1_score, average=weighted, labels=['B-ORG', 'B-MISC', 'B-PER', 'I-PER', 'B-LOC', 'I-ORG', 'I-MISC', 'I-LOC']),\n",
              "                   verbose=1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kekPPLEGS3yS",
        "outputId": "9ae9f355-f359-4026-c3f1-5e77e9ea706c"
      },
      "source": [
        "print('best params:', rs.best_params_)\n",
        "print('best CV score:', rs.best_score_)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "best params: {'c2': 0.025, 'c1': 0.125}\n",
            "best CV score: 0.8427853831671083\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rLnIMV1vS5zM",
        "outputId": "cf1065ca-0ac9-40b1-9a2c-156f6ba51c39"
      },
      "source": [
        "crf = rs.best_estimator_\n",
        "y_pred = crf.predict(X_test)\n",
        "print(metrics.flat_classification_report(\n",
        "    y_test, y_pred, labels=labels, digits=3\n",
        "))"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "       B-ORG      0.767     0.722     0.744      1661\n",
            "      B-MISC      0.804     0.754     0.778       702\n",
            "       B-PER      0.822     0.852     0.836      1617\n",
            "       I-PER      0.870     0.949     0.908      1156\n",
            "       B-LOC      0.854     0.811     0.832      1668\n",
            "       I-ORG      0.673     0.735     0.703       835\n",
            "      I-MISC      0.664     0.657     0.660       216\n",
            "       I-LOC      0.742     0.603     0.665       257\n",
            "\n",
            "   micro avg      0.800     0.797     0.799      8112\n",
            "   macro avg      0.774     0.760     0.766      8112\n",
            "weighted avg      0.800     0.797     0.798      8112\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-HfeKciQS8lN",
        "outputId": "e621736b-f881-473c-f92d-81b1ab01f21f"
      },
      "source": [
        "from collections import Counter\n",
        "\n",
        "def print_transitions(trans_features):\n",
        "    for (label_from, label_to), weight in trans_features:\n",
        "        print(\"%-6s -> %-7s %0.6f\" % (label_from, label_to, weight))\n",
        "\n",
        "print(\"Top likely transitions:\")\n",
        "print_transitions(Counter(crf.transition_features_).most_common(5))"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Top likely transitions:\n",
            "I-MISC -> I-MISC  6.167986\n",
            "B-MISC -> I-MISC  5.679625\n",
            "I-ORG  -> I-ORG   4.966390\n",
            "B-LOC  -> I-LOC   4.908322\n",
            "B-ORG  -> I-ORG   4.836225\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "etDqHA3DSbxy",
        "outputId": "8c2dc31e-5470-494c-8264-9c2f0b25e569"
      },
      "source": [
        "def print_state_features(state_features):\n",
        "    for (attr, label), weight in state_features:\n",
        "        print(\"%0.6f %-8s %s\" % (weight, label, attr))\n",
        "\n",
        "print(\"Top positive:\")\n",
        "print_state_features(Counter(crf.state_features_).most_common(5))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Top positive:\n",
            "10.704585 B-LOC    word.lower():chester-le-street\n",
            "8.677321 B-ORG    word.lower():sungard\n",
            "8.176832 I-LOC    -1:word.lower():colo\n",
            "8.140300 I-LOC    -1:word.lower():wisc\n",
            "8.034788 O        word.lower():minister\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}