Text_Classification_Compare_Models.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/amittian/text_classification_multilabel/blob/main/Text_Classification_Compare_Models.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9DAcH5KPApU7"
      },
      "source": [
        "<h1><center>Text Classification Model Selection</center></h1>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XdSrv7DAApU-"
      },
      "source": [],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XXqOUmYcApVI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 344
        },
        "outputId": "88f1b899-d7da-496e-b584-823704c8bebc"
      },
      "source": [
        "#!pip install stanza\n",
        "#!pip install fasttext\n",
        "#!pip install googletrans\n",
        "!pip install scikit-learn==0.23"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting scikit-learn==0.23\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4b/cd/af7469eb9d2b2ba428206273585f0272b6094ae915aed0b1ca99eace56df/scikit_learn-0.23.0-cp36-cp36m-manylinux1_x86_64.whl (7.3MB)\n",
            "\u001b[K     |████████████████████████████████| 7.3MB 25kB/s \n",
            "\u001b[?25hCollecting threadpoolctl>=2.0.0\n",
            "  Downloading https://files.pythonhosted.org/packages/db/09/cab2f398e28e9f183714afde872b2ce23629f5833e467b151f18e1e08908/threadpoolctl-2.0.0-py3-none-any.whl\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scikit-learn==0.23) (1.18.4)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn==0.23) (0.14.1)\n",
            "Requirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.6/dist-packages (from scikit-learn==0.23) (1.4.1)\n",
            "Installing collected packages: threadpoolctl, scikit-learn\n",
            "  Found existing installation: scikit-learn 0.22.2.post1\n",
            "    Uninstalling scikit-learn-0.22.2.post1:\n",
            "      Successfully uninstalled scikit-learn-0.22.2.post1\n",
            "Successfully installed scikit-learn-0.23.0 threadpoolctl-2.0.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "sklearn"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p3I1rGuOApVS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "ef84ab42-979a-4454-d102-06dd9655df16"
      },
      "source": [
        "import time\n",
        "import glob\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import os\n",
        "import random\n",
        "import stanza\n",
        "\n",
        "from custom_preprocessing import CustomPreProcessing\n",
        "from custom_preprocessing import PreProcessing\n",
        "\n",
        "\n",
        "import sklearn\n",
        "from sklearn.utils import class_weight\n",
        "from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "from sklearn import decomposition, ensemble\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn import tree\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import cohen_kappa_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.model_selection import cross_validate\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "from sklearn.metrics import matthews_corrcoef\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.metrics import balanced_accuracy_score, recall_score, f1_score\n",
        "from sklearn.metrics import make_scorer\n",
        "\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing import sequence\n",
        "\n",
        "from keras.utils import np_utils\n",
        "\n",
        "from xgboost import XGBClassifier\n",
        "\n",
        "\n",
        "import string\n",
        "import fasttext\n",
        "import fasttext.util\n",
        "from tqdm import tqdm\n",
        "\n",
        "# ---- Call tqdm to see progress bar with pandas\n",
        "tqdm().pandas()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n",
            "0it [00:00, ?it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3tByww42ApVe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "d49897d9-9d8a-4ca5-9080-e85f6b4bb9af"
      },
      "source": [
        "print(sklearn.__version__)\n",
        "print(tf.__version__)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.23.0\n",
            "2.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qC12DNK2ApVm"
      },
      "source": [
        "def tn(y_true, y_pred): return confusion_matrix(y_true, y_pred)[0, 0]\n",
        "def fp(y_true, y_pred): return confusion_matrix(y_true, y_pred)[0, 1]\n",
        "def fn(y_true, y_pred): return confusion_matrix(y_true, y_pred)[1, 0]\n",
        "def tp(y_true, y_pred): return confusion_matrix(y_true, y_pred)[1, 1]\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YhNfJUZgApVw"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_cK5zyHeApVy"
      },
      "source": [
        "<center><h2>Parameters</h2></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DqSpoV7fApV0"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jHs2R14UApV2"
      },
      "source": [
        "This part allows you to determine the text column to classify as well as the label column."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W91oAxWGApV4"
      },
      "source": [
        "# choose columns for text and label\n",
        "TEXT = \"text\"\n",
        "LABEL = \"label\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8tqKT8BuApWJ"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oH9b4LEZApWK"
      },
      "source": [
        "<center><h2>List of Models</h2></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eL_GfXmGApWM"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LrvCreHAApWN"
      },
      "source": [
        "# List of paramters for the notebook, choose option like and model to run\n",
        "save_results           = True\n",
        "lang                   = False\n",
        "sample                 = False\n",
        "multinomial_naive_bayes= True\n",
        "logistic_regression    = True\n",
        "svm_model              = False\n",
        "k_nn_model             = False\n",
        "sgd                    = True\n",
        "random_forest          = True\n",
        "gradient_boosting      = True\n",
        "xgboost_classifier     = True\n",
        "shallow_network        = True\n",
        "deep_nn                = True\n",
        "rnn                    = True\n",
        "lstm                   = True\n",
        "cnn                    = True\n",
        "gru                    = True\n",
        "cnn_lstm               = True\n",
        "cnn_gru                = True\n",
        "bidirectional_rnn      = True\n",
        "bidirectional_lstm     = True\n",
        "bidirectional_gru      = True\n",
        "rcnn                   = True\n",
        "pre_trained            = False"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "59t_2nqZApWV"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "20Zu5k2tApWX"
      },
      "source": [
        "<center><h2>List of Metrics for the Model Selection</h2></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dFjNELKFApWZ"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sD8UdKnTApWa"
      },
      "source": [
        "# Dict of metrics to use in the model selection\n",
        "score_metrics = {'acc': accuracy_score,\n",
        "               'balanced_accuracy': balanced_accuracy_score,\n",
        "               'prec': precision_score,\n",
        "               'recall': recall_score,\n",
        "               'f1-score': f1_score,\n",
        "               'tp': tp, 'tn': tn,\n",
        "               'fp': fp, 'fn': fn,\n",
        "               'cohens_kappa':cohen_kappa_score,\n",
        "               'matthews_corrcoef':matthews_corrcoef,\n",
        "               \"roc_auc\":roc_auc_score}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yhmFZ8rnApWl"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hocw8IVRApWn"
      },
      "source": [
        "<center><i><h1>Sand Box to Load Data</h1></i></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GLPtL-dGApWo"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZoQ1y86pApWp"
      },
      "source": [
        "The sandbox is the working area of your data if it has not been processed before using the pipe"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZM3fulG8ApWr",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "1b2bf10a-3fb3-4fae-db39-2d226e9888de"
      },
      "source": [
        "# download IMDB data\n",
        "!wget http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-05-16 19:53:01--  http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n",
            "Resolving ai.stanford.edu (ai.stanford.edu)... 171.64.68.10\n",
            "Connecting to ai.stanford.edu (ai.stanford.edu)|171.64.68.10|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 84125825 (80M) [application/x-gzip]\n",
            "Saving to: ‘aclImdb_v1.tar.gz’\n",
            "\n",
            "aclImdb_v1.tar.gz   100%[===================>]  80.23M  19.5MB/s    in 7.7s    \n",
            "\n",
            "2020-05-16 19:53:09 (10.4 MB/s) - ‘aclImdb_v1.tar.gz’ saved [84125825/84125825]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fAv1iW-sDRzx"
      },
      "source": [
        "!tar -xzf aclImdb_v1.tar.gz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4no2lknTApWy"
      },
      "source": [
        "def load_imdb_sentiment_analysis_dataset(data_path, seed=123):\n",
        "    \"\"\"Loads the IMDb movie reviews sentiment analysis dataset.\n",
        "\n",
        "    # Arguments\n",
        "        data_path: string, path to the data directory.\n",
        "        seed: int, seed for randomizer.\n",
        "\n",
        "    # Returns\n",
        "        A tuple of training and validation data.\n",
        "        Number of training samples: 25000\n",
        "        Number of test samples: 25000\n",
        "        Number of categories: 2 (0 - negative, 1 - positive)\n",
        "\n",
        "    # References\n",
        "        Mass et al., http://www.aclweb.org/anthology/P11-1015\n",
        "\n",
        "        Download and uncompress archive from:\n",
        "        http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n",
        "    \"\"\"\n",
        "    imdb_data_path = os.path.join(data_path, 'aclImdb')\n",
        "\n",
        "    # Load the training data\n",
        "    train_texts = []\n",
        "    train_labels = []\n",
        "    for category in ['pos', 'neg']:\n",
        "        train_path = os.path.join(imdb_data_path, 'train', category)\n",
        "        for fname in tqdm(sorted(os.listdir(train_path))):\n",
        "            if fname.endswith('.txt'):\n",
        "                with open(os.path.join(train_path, fname)) as f:\n",
        "                    train_texts.append(f.read())\n",
        "                train_labels.append(0 if category == 'neg' else 1)\n",
        "    print(\"\\nTrain done\\n\")\n",
        "    # Load the validation data.\n",
        "    test_texts = []\n",
        "    test_labels = []\n",
        "    for category in ['pos', 'neg']:\n",
        "        test_path = os.path.join(imdb_data_path, 'test', category)\n",
        "        for fname in tqdm(sorted(os.listdir(test_path))):\n",
        "            if fname.endswith('.txt'):\n",
        "                with open(os.path.join(test_path, fname)) as f:\n",
        "                    test_texts.append(f.read())\n",
        "                test_labels.append(0 if category == 'neg' else 1)\n",
        "    print(\"\\nTest done\\n\")\n",
        "    # Shuffle the training data and labels.\n",
        "    random.seed(seed)\n",
        "    random.shuffle(train_texts)\n",
        "    random.seed(seed)\n",
        "    random.shuffle(train_labels)\n",
        "\n",
        "    return ((train_texts, np.array(train_labels)),\n",
        "            (test_texts, np.array(test_labels)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mEuS707252Cl"
      },
      "source": [
        "# Functions for preprocessing\n",
        "def remove_upper_case( text):\n",
        "        '''\n",
        "        Function to transform upper string in title words\n",
        "        @param text: (str) text\n",
        "        @return: (str) text without upper words\n",
        "        '''\n",
        "        sentences = text.split(\"\\n\")\n",
        "        new_sentences = []\n",
        "        for i in sentences:\n",
        "            words = text.split()\n",
        "            stripped = [w.title() if w.isupper() else w for w in words]\n",
        "            new_sentences.append(\" \".join(stripped))\n",
        "        return \"\\n\".join(new_sentences)\n",
        "\n",
        "  def remove_URL( text):\n",
        "        '''\n",
        "        Function to remove url from text.\n",
        "        @param text: (str) sentence\n",
        "        @return: (str) clean text\n",
        "\n",
        "        '''\n",
        "        url = re.compile(r'https?://\\S+|www\\.\\S+')\n",
        "        return url.sub(r'',text)\n",
        "\n",
        "\n",
        "  def remove_html( text):\n",
        "        '''\n",
        "        Function regex to clean text from html balises.\n",
        "        @param text: (str) sentence\n",
        "        @return: (str) clean text\n",
        "        '''\n",
        "        html=re.compile(r'<.*?>')\n",
        "        return html.sub(r'',text)\n",
        "\n",
        "\n",
        "\n",
        "  def remove_emoji( text):\n",
        "        '''\n",
        "        Function to remove emojis, symbols and pictograms etc from text\n",
        "        @param text: (str) sentences\n",
        "        @return: (str) clean text\n",
        "        '''\n",
        "        emoji_pattern = re.compile(\"[\"\n",
        "                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "                               u\"\\U00002702-\\U000027B0\"\n",
        "                               u\"\\U000024C2-\\U0001F251\"\n",
        "                               \"]+\", flags=re.UNICODE)\n",
        "        return emoji_pattern.sub(r'', text)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MJ2EAQuvApW5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        },
        "outputId": "fcf1946a-d97d-43b7-d51f-3ae1c1536411"
      },
      "source": [
        "#%%script false --no-raise-error\n",
        "%%time\n",
        "(x_train, y_train), (x_test, y_test) = load_imdb_sentiment_analysis_dataset(\".\")\n",
        "\n",
        "df = pd.DataFrame(data=[x_train, y_train], index=[\"text\", \"label\"]).T\n",
        "df = df.append(pd.DataFrame(data=[x_test, y_test], index=[\"text\", \"label\"]).T)\n",
        "\n",
        "df[TEXT] = df[TEXT].apply(remove_upper_case)\n",
        "df[TEXT] = df[TEXT].apply(remove_URL)\n",
        "df[TEXT] = df[TEXT].apply(remove_html)\n",
        "df[TEXT] = df[TEXT].apply(remove_emoji)\n",
        "\n",
        "print(df.head())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 12500/12500 [00:00<00:00, 36355.21it/s]\n",
            "100%|██████████| 12500/12500 [00:00<00:00, 35571.19it/s]\n",
            " 27%|██▋       | 3339/12500 [00:00<00:00, 33384.30it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Train done\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 12500/12500 [00:00<00:00, 35090.25it/s]\n",
            "100%|██████████| 12500/12500 [00:00<00:00, 37395.40it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test done\n",
            "\n",
            "                                                text label\n",
            "0  Possible SPOILERSThe Spy Who Shagged Me is a m...     0\n",
            "1  The long list of \"big\" names in this flick (in...     0\n",
            "2  Bette Midler showcases her talents and beauty ...     1\n",
            "3  Great movie when I saw it. Have to say one of ...     1\n",
            "4  Although it's most certainly politically incor...     1\n",
            "CPU times: user 9.23 s, sys: 757 ms, total: 9.99 s\n",
            "Wall time: 10 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_4BuQZbkDtRS"
      },
      "source": [
        "So at this moment the data is ready to be classified !"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yWQoeE8QApXB"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swG2mc5UApXC"
      },
      "source": [
        "<center><i><h1>Sart Pipeline</h1></i></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9WMkC8KKApXE"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BSM1cvbFApXF"
      },
      "source": [
        "if lang:\n",
        "    # ---- Language detection of the text\n",
        "    df.loc[:,\"language\"] = df[TEXT].progress_apply(preproc.func_detect_lang_google)\n",
        "    # ---- Extract most frequent language\n",
        "    language = df.language.value_counts().index.tolist()[0]\n",
        "    print(f\"The language most present in the dataset is {language}\")\n",
        "else:\n",
        "    language=\"en\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aBSYBEfxApXN"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_C9ESeodApXP"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KLeZAby1ApXQ"
      },
      "source": [
        "<center><h3>Prepare data for ML Classic</h3></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T9iOMndEApXR"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kN7vGu2CApXS"
      },
      "source": [
        "if sample:\n",
        "    df_save = df.copy()\n",
        "    df = df.sample(5000, random_state=42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AJeYzkuNApXa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e7625225-cc97-4e95-aea2-9dfe1e29f73c"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50000, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Vs-Sl_mApXi"
      },
      "source": [
        "# ---- Load stopwords\n",
        "if language==\"fr\":\n",
        "    stop_word = np.loadtxt(\"stopwords-fr.txt\", dtype=str)\n",
        "if language==\"en\":\n",
        "    stop_word = np.loadtxt(\"stopwords_en.txt\", dtype=str)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mxxZnvUWEV4K"
      },
      "source": [
        "To One-Hot Encoding is better without stopwords"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lfSG9cy7ApXo",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b9b9b54e-d9c6-4cf6-96f2-4ce72f0cfe78"
      },
      "source": [
        "df.loc[:,TEXT+\"_sw\"] = df.loc[:,TEXT].progress_apply(lambda x : preproc.func_remove_stop_words(x, stop_word))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 50000/50000 [00:59<00:00, 836.71it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DZNmasgLApXv"
      },
      "source": [
        "if df[TEXT+\"_sw\"].isnull().sum()>0:\n",
        "    print(\"Empty text\")\n",
        "    df[TEXT+\"_sw\"][df[TEXT+\"_sw\"].isnull()] = \"empty_text\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vVcwGBShApX1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "afbc2426-693e-4c03-d313-e91641072576"
      },
      "source": [
        "df[LABEL].isnull().sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WOAIyXB2ApX8"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NdrZPec1ApX9"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WuOx6izxApX-"
      },
      "source": [
        "<h1><center>Machine Learning</center></h1>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n4D9dm4sApX_"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6BQPIbi6ApYA"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xil1On5MApYE"
      },
      "source": [
        "# split the dataset into training and validation datasets\n",
        "# ML classic\n",
        "train_x_sw, valid_x_sw, y_train_sw, y_valid_sw = model_selection.train_test_split(df[TEXT+\"_sw\"], df[LABEL], random_state=42, stratify=df[LABEL], test_size=0.2)\n",
        "\n",
        "# For Embeddings\n",
        "train_x, valid_x, y_train, y_valid = model_selection.train_test_split(df[TEXT], df[LABEL], random_state=42, stratify=df[LABEL], test_size=0.2)\n",
        "\n",
        "# label encode the target variable\n",
        "encoder = preprocessing.LabelEncoder()\n",
        "train_y_sw = encoder.fit_transform(y_train_sw)\n",
        "valid_y_sw = encoder.fit_transform(y_valid_sw)\n",
        "train_y = encoder.fit_transform(y_train)\n",
        "valid_y = encoder.fit_transform(y_valid)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t9ur2FCzApYL"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "24VWbA3KApYM"
      },
      "source": [
        "<center><h3>Classes Weight</h3></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JAq5CnRoApYO"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G25twHAnApYP"
      },
      "source": [
        "# Compute the class weight with sklearn\n",
        "class_weights = class_weight.compute_class_weight('balanced',\n",
        "                                                 np.unique(y_train),\n",
        "                                                 y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P8XnusVvApYW"
      },
      "source": [
        "print(*[f'Class weight: {round(i[0],4)}\\tclass: {i[1]}' for i in zip(class_weights, np.unique(y_train))], sep='\\n')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1kC2sClrApYf"
      },
      "source": [
        "# Determined if the dataset is balanced or imbalanced\n",
        "ratio = np.min(df.label.value_counts()) / np.max(df.label.value_counts())\n",
        "if ratio > 0.1:      # Ratio 1:10 -> limite blanced / imbalanced\n",
        "    balanced = True\n",
        "    print(f\"\\nThe dataset is balanced (ratio={round(ratio, 3)})\")\n",
        "else:\n",
        "    balanced = False\n",
        "    print(f\"\\nThe dataset is imbalanced (ratio={round(ratio, 3)})\")\n",
        "    #from imblearn.over_sampling import ADASYN\n",
        "    # put class for debalanced data\n",
        "    # in progress"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gCUXZ_EYApYm"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y500uSVLApYo"
      },
      "source": [
        "<h2>Save Unique Labels</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b9-nh1C0ApYp"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BzSx30snApYr"
      },
      "source": [
        "# Keep the unique label corresponding to their encoding correspondance\n",
        "labels = df[LABEL].unique()\n",
        "test=pd.DataFrame(data=np.transpose([labels,encoder.fit_transform(labels)]), columns=[\"labels\", \"encoding\"]).sort_values(by=[\"encoding\"])\n",
        "labels=test.labels.tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VJfQwbzUApYy"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1gAY9ECsApYz"
      },
      "source": [
        "<h2>DataFrame for the results</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2rFEoZkJApY0"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1QZ30n3dApY2"
      },
      "source": [
        "df_results = pd.DataFrame()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LvGWuDv5ApY9"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lOhlksIBApY_"
      },
      "source": [
        "<h3>One-Hot encoding (CountVectorizing)</h3>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5qdey2dhApZA"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ICDOeifApZC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "67afa012-81dc-4d26-906d-2f3323ca60ac"
      },
      "source": [
        "%%time\n",
        "# create a count vectorizer object\n",
        "count_vect = CountVectorizer(analyzer='word', token_pattern=r'\\w{1,}')\n",
        "count_vect.fit(df[TEXT]+\"_sw\")\n",
        "\n",
        "# transform the training and validation data using count vectorizer object\n",
        "xtrain_count =  count_vect.transform(train_x_sw)\n",
        "xvalid_count =  count_vect.transform(valid_x_sw)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 11.2 s, sys: 253 ms, total: 11.4 s\n",
            "Wall time: 11.5 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pzWn5meDApZH"
      },
      "source": [
        "#xtrain_tfidf.toarray()[0][xtrain_tfidf.toarray()[0]  >0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EDm6S9NzApZM"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-0xrGS2qApZO"
      },
      "source": [
        "<h3>TF-IDF</h3>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "chEZ4GdxApZP"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O6gF53DKApZR",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "21a48655-f459-4443-99d8-ca679477b666"
      },
      "source": [
        "%%time\n",
        "# word level tf-idf\n",
        "tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', max_features=10000)\n",
        "tfidf_vect.fit(df[TEXT])\n",
        "xtrain_tfidf =  tfidf_vect.transform(train_x_sw)\n",
        "xvalid_tfidf =  tfidf_vect.transform(valid_x_sw)\n",
        "print(\"word level tf-idf done\")\n",
        "# ngram level tf-idf\n",
        "tfidf_vect_ngram = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', ngram_range=(2,3), max_features=10000)\n",
        "tfidf_vect_ngram.fit(df[TEXT])\n",
        "xtrain_tfidf_ngram =  tfidf_vect_ngram.transform(train_x_sw)\n",
        "xvalid_tfidf_ngram =  tfidf_vect_ngram.transform(valid_x_sw)\n",
        "print(\"ngram level tf-idf done\")\n",
        "# characters level tf-idf\n",
        "tfidf_vect_ngram_chars = TfidfVectorizer(analyzer='char',  ngram_range=(2,3), max_features=10000) #token_pattern=r'\\w{1,}',\n",
        "tfidf_vect_ngram_chars.fit(df[TEXT])\n",
        "xtrain_tfidf_ngram_chars =  tfidf_vect_ngram_chars.transform(train_x_sw)\n",
        "xvalid_tfidf_ngram_chars =  tfidf_vect_ngram_chars.transform(valid_x_sw)\n",
        "print(\"characters level tf-idf done\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "word level tf-idf done\n",
            "ngram level tf-idf done\n",
            "characters level tf-idf done\n",
            "CPU times: user 2min 32s, sys: 2.35 s, total: 2min 35s\n",
            "Wall time: 2min 35s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0pP_ezk7ApZY"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f62-2odhApZa"
      },
      "source": [
        "<h2>Load Pre-Trained model fastText</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y2wexAMFApZb"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oxAjWfOBApZc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        },
        "outputId": "4df3a8cd-129e-4936-809b-03e86ca888a1"
      },
      "source": [
        "%%time\n",
        "if language==\"fr\":\n",
        "    !wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.fr.300.bin.gz\n",
        "    !gunzip cc.fr.300.bin.gz\n",
        "    pretrained = fasttext.FastText.load_model('cc.fr.300.bin')\n",
        "if language==\"en\":\n",
        "    !wget https://dl.fbaipublicfiles.com/fasttext/vectors-english/crawl-300d-2M-subword.zip\n",
        "    !unzip crawl-300d-2M-subword.zip\n",
        "    pretrained = fasttext.FastText.load_model('crawl-300d-2M-subword.bin')\n",
        ""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-05-16 20:02:26--  https://dl.fbaipublicfiles.com/fasttext/vectors-english/crawl-300d-2M-subword.zip\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.75.142, 104.22.74.142, 2606:4700:10::6816:4a8e, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.75.142|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 5828358084 (5.4G) [application/zip]\n",
            "Saving to: ‘crawl-300d-2M-subword.zip’\n",
            "\n",
            "crawl-300d-2M-subwo 100%[===================>]   5.43G  14.3MB/s    in 6m 31s  \n",
            "\n",
            "2020-05-16 20:08:58 (14.2 MB/s) - ‘crawl-300d-2M-subword.zip’ saved [5828358084/5828358084]\n",
            "\n",
            "Archive:  crawl-300d-2M-subword.zip\n",
            "  inflating: crawl-300d-2M-subword.vec  \n",
            "  inflating: crawl-300d-2M-subword.bin  \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 6 s, sys: 6.93 s, total: 12.9 s\n",
            "Wall time: 11min 51s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f9HBX4xRApZk"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "up3qxcEIApZl"
      },
      "source": [
        "<h2>Word Embeddings</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YoBQbmPmApZo"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZiY_sh0_ApZq",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "441dabdb-256a-41ce-e144-6da8a98ce199"
      },
      "source": [
        "%%time\n",
        "# create a tokenizer\n",
        "token = Tokenizer()\n",
        "token.fit_on_texts(df[TEXT])\n",
        "word_index = token.word_index\n",
        "\n",
        "# convert text to sequence of tokens and pad them to ensure equal length vectors\n",
        "train_seq_x = sequence.pad_sequences(token.texts_to_sequences(train_x), maxlen=300)\n",
        "valid_seq_x = sequence.pad_sequences(token.texts_to_sequences(valid_x), maxlen=300)\n",
        "\n",
        "# create token-embedding mapping\n",
        "embedding_matrix = np.zeros((len(word_index) + 1, 300))\n",
        "words = []\n",
        "for word, i in tqdm(word_index.items()):\n",
        "    embedding_vector = pretrained.get_word_vector(word) #embeddings_index.get(word)\n",
        "    words.append(word)\n",
        "    if embedding_vector is not None:\n",
        "        embedding_matrix[i] = embedding_vector\n",
        ""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 125305/125305 [00:01<00:00, 78858.27it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 16.8 s, sys: 348 ms, total: 17.1 s\n",
            "Wall time: 17 s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MVXZeZFuApZx"
      },
      "source": [
        "#words[1], embedding_matrix[1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tj0I6dneApZ5"
      },
      "source": [
        "def report(clf, x, y, name='classifier', cv=5, dict_scoring=None, fit_params=None):\n",
        "    #print(dict_scoring)\n",
        "    if dict_scoring!=None:\n",
        "        score = dict_scoring.copy()\n",
        "        for i in score.keys():\n",
        "            score[i] = make_scorer(score[i])\n",
        "\n",
        "    #if clf==XGBClassifier():\n",
        "    scores = cross_validate(clf, x, y, scoring=score,\n",
        "                         cv=cv, return_train_score=False, n_jobs=-1,  fit_params=fit_params)\n",
        "\n",
        "    index = []\n",
        "    value = []\n",
        "    index.append(\"Model\")\n",
        "    value.append(name)\n",
        "    for i in scores:\n",
        "        if i == \"estimator\":\n",
        "            continue\n",
        "        for j in enumerate(scores[i]):\n",
        "            index.append(i+\"_cv\"+str(j[0]+1))\n",
        "            value.append(j[1])\n",
        "        #if any(x in i for x in scoring.keys()):\n",
        "\n",
        "        index.append(i+\"_mean\")\n",
        "        value.append(np.mean(scores[i]))\n",
        "        index.append(i+\"_std\")\n",
        "        value.append(np.std(scores[i]))\n",
        "\n",
        "    return pd.DataFrame(data=value, index=index).T"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v5faeHAsApaA"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3OG15GOlApaC"
      },
      "source": [
        "<center><h2>Multinomial Naive Bayes</h2></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ed4mKShEApaE"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_b7DoCfKApaI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "7d184ec6-3e31-437a-e3c6-5562fccdb7ec"
      },
      "source": [
        "%%time\n",
        "if multinomial_naive_bayes:\n",
        "    df_results = df_results.append(report(naive_bayes.MultinomialNB(), xtrain_count,train_y_sw, name='NB_Count_Vectors', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(naive_bayes.MultinomialNB(), xtrain_tfidf,train_y, name='NB_WordLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(naive_bayes.MultinomialNB(), xtrain_tfidf_ngram,train_y, name='NB_N-Gram_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(naive_bayes.MultinomialNB(), xtrain_tfidf_ngram_chars,train_y, name='NB_CharLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(naive_bayes.MultinomialNB(), train_seq_x,train_y, name='NB_Words', cv=5, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/joblib/externals/loky/process_executor.py:706: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
            "  \"timeout or by a memory leak.\", UserWarning\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 421 ms, sys: 930 ms, total: 1.35 s\n",
            "Wall time: 8.34 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5-qZmzFYApaO"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oFInztXRApaP"
      },
      "source": [
        "<center><h2>Logistic Regression</h2></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "30VZtmeNApaR"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5D2ETDg_ApaS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "d87921d1-e988-45d9-9ef8-75247b42315e"
      },
      "source": [
        "%%time\n",
        "if logistic_regression:\n",
        "    df_results = df_results.append(report(linear_model.LogisticRegression(max_iter=1000), xtrain_count,train_y_sw, name='LR_Count_Vectors', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(linear_model.LogisticRegression(max_iter=1000), xtrain_tfidf,train_y, name='LR_WordLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(linear_model.LogisticRegression(max_iter=1000), xtrain_tfidf_ngram,train_y, name='LR_N-Gram_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(linear_model.LogisticRegression(max_iter=1000), xtrain_tfidf_ngram_chars,train_y, name='LR_CharLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(linear_model.LogisticRegression(max_iter=1000), train_seq_x,train_y, name='LR_Words', cv=5, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/joblib/externals/loky/process_executor.py:706: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
            "  \"timeout or by a memory leak.\", UserWarning\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 374 ms, sys: 372 ms, total: 746 ms\n",
            "Wall time: 1min 31s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1dmVp_iXApaZ"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f1dPsr0wApaa"
      },
      "source": [
        "<center><h2>SVM</h2></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H-n0ya-JApab"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0VgAE767Apad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "0bc6e631-a5c3-4b5b-9662-66528d32a111"
      },
      "source": [
        "%%time\n",
        "if svm_model:\n",
        "    df_results = df_results.append(report(svm.SVC(), xtrain_count,train_y_sw, name='SVM_Count_Vectors', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(svm.SVC(), xtrain_tfidf,train_y, name='SVM_WordLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(svm.SVC(), xtrain_tfidf_ngram,train_y, name='SVM_N-Gram_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(svm.SVC(), xtrain_tfidf_ngram_chars,train_y, name='SVM_CharLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(svm.SVC(), train_seq_x,train_y, name='SVM_Words', cv=5, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 4 µs, sys: 0 ns, total: 4 µs\n",
            "Wall time: 5.48 µs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KGHctKvEApaj"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QywTzRBPApam"
      },
      "source": [
        "<center><h2>k-NN</h2></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0QVM4i1sApan"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JXXj_BGCApan",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "635a6f00-ee83-43cc-c0a0-dc8395c7fcb7"
      },
      "source": [
        "%%time\n",
        "if k_nn_model:\n",
        "    df_results = df_results.append(report(KNeighborsClassifier(n_neighbors=20, weights='distance', n_jobs=-1), xtrain_count,train_y_sw, name='kNN_Count_Vectors', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(KNeighborsClassifier(n_neighbors=20, weights='distance', n_jobs=-1), xtrain_tfidf,train_y, name='kNN_WordLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(KNeighborsClassifier(n_neighbors=20, weights='distance', n_jobs=-1), xtrain_tfidf_ngram,train_y, name='kNN_N-Gram_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(KNeighborsClassifier(n_neighbors=20, weights='distance', n_jobs=-1), xtrain_tfidf_ngram_chars,train_y, name='kNN_CharLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(KNeighborsClassifier(n_neighbors=20, weights='distance', n_jobs=-1), train_seq_x,train_y, name='kNN_Words', cv=5, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 3 µs, sys: 0 ns, total: 3 µs\n",
            "Wall time: 6.44 µs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UW6hPydNApas"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cS4sN36sApat"
      },
      "source": [
        "<center><h2>RandomForest</h2></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eXMw3YsiApau"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bbd238jUApav",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "d0626747-3267-43cf-f97e-70fec4fcf5b7"
      },
      "source": [
        "%%time\n",
        "if random_forest:\n",
        "    df_results = df_results.append(report(ensemble.RandomForestClassifier(bootstrap=True,min_impurity_decrease=1e-7,n_jobs=-1, random_state=42), xtrain_count,train_y_sw, name='RF_Count_Vectors', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(ensemble.RandomForestClassifier(bootstrap=True,min_impurity_decrease=1e-7,n_jobs=-1, random_state=42), xtrain_tfidf,train_y, name='RF_WordLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(ensemble.RandomForestClassifier(bootstrap=True,min_impurity_decrease=1e-7,n_jobs=-1, random_state=42), xtrain_tfidf_ngram,train_y, name='RF_N-Gram_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(ensemble.RandomForestClassifier(bootstrap=True,min_impurity_decrease=1e-7,n_jobs=-1, random_state=42), xtrain_tfidf_ngram_chars,train_y, name='RF_CharLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(ensemble.RandomForestClassifier(bootstrap=True,min_impurity_decrease=1e-7,n_jobs=-1, random_state=42), train_seq_x,train_y, name='RF_Words', cv=5, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/joblib/externals/loky/process_executor.py:706: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
            "  \"timeout or by a memory leak.\", UserWarning\n",
            "/usr/local/lib/python3.6/dist-packages/joblib/externals/loky/process_executor.py:706: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
            "  \"timeout or by a memory leak.\", UserWarning\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 496 ms, sys: 1.09 s, total: 1.58 s\n",
            "Wall time: 32min 48s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0coqjipFApa1"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wFUr_mymApa2"
      },
      "source": [
        "<center><h2>Stochastic Descent</h2></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a-6esWXkApa3"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c9HTipuPApa6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "f917f7ee-9411-4f2d-e55f-8061213e054e"
      },
      "source": [
        "%%time\n",
        "if sgd:\n",
        "    df_results = df_results.append(report(SGDClassifier(loss='modified_huber', max_iter=1000, tol=1e-3,   n_iter_no_change=10, early_stopping=True, n_jobs=-1 ), xtrain_count,train_y_sw, name='SGD_Count_Vectors', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(SGDClassifier(loss='modified_huber', max_iter=1000, tol=1e-3,   n_iter_no_change=10, early_stopping=True, n_jobs=-1 ), xtrain_tfidf,train_y, name='SGD_WordLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(SGDClassifier(loss='modified_huber', max_iter=1000, tol=1e-3,   n_iter_no_change=10, early_stopping=True, n_jobs=-1 ), xtrain_tfidf_ngram,train_y, name='SGD_N-Gram_Vectors', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(SGDClassifier(loss='modified_huber', max_iter=1000, tol=1e-3,   n_iter_no_change=10, early_stopping=True, n_jobs=-1 ), xtrain_tfidf_ngram_chars,train_y, name='SGD_CharLevel_Vectors', cv=5, dict_scoring=score_metrics))\n",
        "    df_results = df_results.append(report(SGDClassifier(loss='modified_huber', max_iter=1000, tol=1e-3,   n_iter_no_change=10, early_stopping=True, n_jobs=-1 ), train_seq_x,train_y, name='SGD_Words', cv=5, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/joblib/externals/loky/process_executor.py:706: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
            "  \"timeout or by a memory leak.\", UserWarning\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 373 ms, sys: 436 ms, total: 809 ms\n",
            "Wall time: 13.3 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H8aMAGO3Apa_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        },
        "outputId": "57e88db0-e1db-439b-c4cd-2e4561276f16"
      },
      "source": [
        "df_results[[\"Model\",\"test_acc_mean\", \"test_prec_mean\", \"test_recall_mean\", \"test_f1-score_mean\", \"test_cohens_kappa_mean\"]].sort_values(by=[\"test_prec_mean\"], ascending=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Model</th>\n",
              "      <th>test_acc_mean</th>\n",
              "      <th>test_prec_mean</th>\n",
              "      <th>test_recall_mean</th>\n",
              "      <th>test_f1-score_mean</th>\n",
              "      <th>test_cohens_kappa_mean</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>SGD_WordLevel_TF-IDF</td>\n",
              "      <td>0.89355</td>\n",
              "      <td>0.886203</td>\n",
              "      <td>0.90315</td>\n",
              "      <td>0.894558</td>\n",
              "      <td>0.7871</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>LR_WordLevel_TF-IDF</td>\n",
              "      <td>0.89505</td>\n",
              "      <td>0.885092</td>\n",
              "      <td>0.908</td>\n",
              "      <td>0.896397</td>\n",
              "      <td>0.7901</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>SGD_Count_Vectors</td>\n",
              "      <td>0.865375</td>\n",
              "      <td>0.884033</td>\n",
              "      <td>0.84325</td>\n",
              "      <td>0.861576</td>\n",
              "      <td>0.73075</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>LR_Count_Vectors</td>\n",
              "      <td>0.8861</td>\n",
              "      <td>0.883098</td>\n",
              "      <td>0.89005</td>\n",
              "      <td>0.886552</td>\n",
              "      <td>0.7722</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NB_Count_Vectors</td>\n",
              "      <td>0.8561</td>\n",
              "      <td>0.874254</td>\n",
              "      <td>0.83185</td>\n",
              "      <td>0.852523</td>\n",
              "      <td>0.7122</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>SGD_CharLevel_Vectors</td>\n",
              "      <td>0.862225</td>\n",
              "      <td>0.87084</td>\n",
              "      <td>0.85325</td>\n",
              "      <td>0.860464</td>\n",
              "      <td>0.72445</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NB_WordLevel_TF-IDF</td>\n",
              "      <td>0.858225</td>\n",
              "      <td>0.858175</td>\n",
              "      <td>0.85835</td>\n",
              "      <td>0.858248</td>\n",
              "      <td>0.71645</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>RF_WordLevel_TF-IDF</td>\n",
              "      <td>0.852</td>\n",
              "      <td>0.855862</td>\n",
              "      <td>0.8466</td>\n",
              "      <td>0.851199</td>\n",
              "      <td>0.704</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>RF_Count_Vectors</td>\n",
              "      <td>0.858125</td>\n",
              "      <td>0.855779</td>\n",
              "      <td>0.86145</td>\n",
              "      <td>0.858595</td>\n",
              "      <td>0.71625</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>LR_CharLevel_TF-IDF</td>\n",
              "      <td>0.86115</td>\n",
              "      <td>0.854156</td>\n",
              "      <td>0.87105</td>\n",
              "      <td>0.86251</td>\n",
              "      <td>0.7223</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NB_CharLevel_TF-IDF</td>\n",
              "      <td>0.807775</td>\n",
              "      <td>0.79995</td>\n",
              "      <td>0.82085</td>\n",
              "      <td>0.810252</td>\n",
              "      <td>0.61555</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>RF_CharLevel_TF-IDF</td>\n",
              "      <td>0.7965</td>\n",
              "      <td>0.795084</td>\n",
              "      <td>0.799</td>\n",
              "      <td>0.797021</td>\n",
              "      <td>0.593</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NB_N-Gram_TF-IDF</td>\n",
              "      <td>0.74655</td>\n",
              "      <td>0.758306</td>\n",
              "      <td>0.72385</td>\n",
              "      <td>0.740668</td>\n",
              "      <td>0.4931</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>LR_N-Gram_TF-IDF</td>\n",
              "      <td>0.757</td>\n",
              "      <td>0.748553</td>\n",
              "      <td>0.774</td>\n",
              "      <td>0.761063</td>\n",
              "      <td>0.514</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>SGD_N-Gram_Vectors</td>\n",
              "      <td>0.751975</td>\n",
              "      <td>0.746322</td>\n",
              "      <td>0.7643</td>\n",
              "      <td>0.754941</td>\n",
              "      <td>0.50395</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>RF_N-Gram_TF-IDF</td>\n",
              "      <td>0.73715</td>\n",
              "      <td>0.743082</td>\n",
              "      <td>0.7249</td>\n",
              "      <td>0.733861</td>\n",
              "      <td>0.4743</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>RF_Words</td>\n",
              "      <td>0.545275</td>\n",
              "      <td>0.547821</td>\n",
              "      <td>0.51905</td>\n",
              "      <td>0.533037</td>\n",
              "      <td>0.09055</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NB_Words</td>\n",
              "      <td>0.506725</td>\n",
              "      <td>0.508228</td>\n",
              "      <td>0.4159</td>\n",
              "      <td>0.457373</td>\n",
              "      <td>0.01345</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>LR_Words</td>\n",
              "      <td>0.5045</td>\n",
              "      <td>0.504058</td>\n",
              "      <td>0.56275</td>\n",
              "      <td>0.531742</td>\n",
              "      <td>0.009</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>SGD_Words</td>\n",
              "      <td>0.500275</td>\n",
              "      <td>0.500219</td>\n",
              "      <td>0.5147</td>\n",
              "      <td>0.505521</td>\n",
              "      <td>0.00055</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                   Model  ... test_cohens_kappa_mean\n",
              "0   SGD_WordLevel_TF-IDF  ...                 0.7871\n",
              "0    LR_WordLevel_TF-IDF  ...                 0.7901\n",
              "0      SGD_Count_Vectors  ...                0.73075\n",
              "0       LR_Count_Vectors  ...                 0.7722\n",
              "0       NB_Count_Vectors  ...                 0.7122\n",
              "0  SGD_CharLevel_Vectors  ...                0.72445\n",
              "0    NB_WordLevel_TF-IDF  ...                0.71645\n",
              "0    RF_WordLevel_TF-IDF  ...                  0.704\n",
              "0       RF_Count_Vectors  ...                0.71625\n",
              "0    LR_CharLevel_TF-IDF  ...                 0.7223\n",
              "0    NB_CharLevel_TF-IDF  ...                0.61555\n",
              "0    RF_CharLevel_TF-IDF  ...                  0.593\n",
              "0       NB_N-Gram_TF-IDF  ...                 0.4931\n",
              "0       LR_N-Gram_TF-IDF  ...                  0.514\n",
              "0     SGD_N-Gram_Vectors  ...                0.50395\n",
              "0       RF_N-Gram_TF-IDF  ...                 0.4743\n",
              "0               RF_Words  ...                0.09055\n",
              "0               NB_Words  ...                0.01345\n",
              "0               LR_Words  ...                  0.009\n",
              "0              SGD_Words  ...                0.00055\n",
              "\n",
              "[20 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ewzUY_MYApbE"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QWJrlqyUApbH"
      },
      "source": [
        "<center><h2>Gradient Boosting</h2></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dYkqWohCApbJ"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eZhOchHTApbK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "db10b8be-0e06-4948-9aff-ef50448e5cd5"
      },
      "source": [
        "%%time\n",
        "if gradient_boosting:\n",
        "    df_results = df_results.append(report(ensemble.GradientBoostingClassifier(n_estimators=1000,\n",
        "                                               validation_fraction=0.2,\n",
        "                                               n_iter_no_change=10, tol=0.01,\n",
        "                                               random_state=0, verbose=0 ), xtrain_count,train_y_sw, name='GB_Count_Vectors', cv=5, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 107 ms, sys: 30.2 ms, total: 138 ms\n",
            "Wall time: 15min 21s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FJDc2aYTApbP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "4e79cb04-7ea1-4c62-9084-3fcfd16641a0"
      },
      "source": [
        "%%time\n",
        "if gradient_boosting:\n",
        "    df_results = df_results.append(report(ensemble.GradientBoostingClassifier(n_estimators=1000,\n",
        "                                               validation_fraction=0.2,\n",
        "                                               n_iter_no_change=10, tol=0.01,\n",
        "                                               random_state=0, verbose=0 ), xtrain_tfidf,train_y, name='GB_WordLevel_TF-IDF', cv=5, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 117 ms, sys: 30.1 ms, total: 147 ms\n",
            "Wall time: 7min 44s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0t4CC77VApbY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "c805f311-b08b-47c1-9788-d1d32878635b"
      },
      "source": [
        "%%time\n",
        "if gradient_boosting:\n",
        "    df_results = df_results.append(report(ensemble.GradientBoostingClassifier(n_estimators=1000,\n",
        "                                               validation_fraction=0.2,\n",
        "                                               n_iter_no_change=10, tol=0.01,\n",
        "                                               random_state=0, verbose=0 ), xtrain_tfidf_ngram,train_y, name='GB_N-Gram_TF-IDF', cv=5, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 53 ms, sys: 9.86 ms, total: 62.9 ms\n",
            "Wall time: 35.6 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "btGtk1nhApbk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 395
        },
        "outputId": "089b904e-b2f5-460a-b128-4863f7076da8"
      },
      "source": [
        "%%time\n",
        "if gradient_boosting:\n",
        "    df_results = df_results.append(report(ensemble.GradientBoostingClassifier(n_estimators=1000,\n",
        "                                               validation_fraction=0.2,\n",
        "                                               n_iter_no_change=10, tol=0.01,\n",
        "                                               random_state=0, verbose=0 ), xtrain_tfidf_ngram_chars,train_y, name='GB_CharLevel_TF-IDF', cv=5, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TerminatedWorkerError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTerminatedWorkerError\u001b[0m                     Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-42-e2a550b71026>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'time'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"if gradient_boosting:\\n    df_results = df_results.append(report(ensemble.GradientBoostingClassifier(n_estimators=1000,\\n                                               validation_fraction=0.2,\\n                                               n_iter_no_change=10, tol=0.01,\\n                                               random_state=0, verbose=0 ), xtrain_tfidf_ngram_chars,train_y, name='GB_CharLevel_TF-IDF', cv=5, dict_scoring=score_metrics))\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m   2115\u001b[0m             \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvar_expand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2116\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2117\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2118\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<decorator-gen-60>\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m         \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/magics/execution.py\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n\u001b[1;32m   1191\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1192\u001b[0m             \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1193\u001b[0;31m             \u001b[0mexec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglob\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_ns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1194\u001b[0m             \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32m<ipython-input-28-23dc97bec1c7>\u001b[0m in \u001b[0;36mreport\u001b[0;34m(clf, x, y, name, cv, dict_scoring, fit_params)\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;31m#if clf==XGBClassifier():\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     scores = cross_validate(clf, x, y, scoring=score,\n\u001b[0;32m---> 10\u001b[0;31m                          cv=cv, return_train_score=False, n_jobs=-1,  fit_params=fit_params)\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     71\u001b[0m                           FutureWarning)\n\u001b[1;32m     72\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36mcross_validate\u001b[0;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score, return_estimator, error_score)\u001b[0m\n\u001b[1;32m    246\u001b[0m             \u001b[0mreturn_times\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_estimator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_estimator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m             error_score=error_score)\n\u001b[0;32m--> 248\u001b[0;31m         for train, test in cv.split(X, y, groups))\n\u001b[0m\u001b[1;32m    249\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m     \u001b[0mzipped_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1015\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1017\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1018\u001b[0m             \u001b[0;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1019\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    907\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    908\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'supports_timeout'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 909\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    910\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    911\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[0;34m(future, timeout)\u001b[0m\n\u001b[1;32m    560\u001b[0m         AsyncResults.get from multiprocessing.\"\"\"\n\u001b[1;32m    561\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 562\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    563\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mLokyTimeoutError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    564\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.6/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    430\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mCancelledError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    431\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mFINISHED\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 432\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    433\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    434\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.6/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    382\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    383\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 384\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    385\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTerminatedWorkerError\u001b[0m: A worker process managed by the executor was unexpectedly terminated. This could be caused by a segmentation fault while calling the function or by an excessive memory usage causing the Operating System to kill the worker. The exit codes of the workers are {SIGKILL(-9)}"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7yK-OhQAApbu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "0ca764d6-b810-45cf-bbd3-3d0c548c11a5"
      },
      "source": [
        "%%time\n",
        "if gradient_boosting:\n",
        "    df_results = df_results.append(report(ensemble.GradientBoostingClassifier(n_estimators=1000,\n",
        "                                               validation_fraction=0.2,\n",
        "                                               n_iter_no_change=10, tol=0.01,\n",
        "                                               random_state=0, verbose=0 ), train_seq_x,train_y, name='GB_Words', cv=5, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 119 ms, sys: 337 ms, total: 457 ms\n",
            "Wall time: 1min 3s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sY8HWLPWApbz"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jsTrFb_zApb0"
      },
      "source": [
        "<h2>XGBoost Classifier</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mhf9HAqgApb1"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MCb5R2XTApb2"
      },
      "source": [
        "All the XGBoost have early stopping implemented with 10 rounds"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vHVDrR0AApb3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "460b08cb-de76-4611-b4e3-9dacec343ec9"
      },
      "source": [
        "%%time\n",
        "if xgboost_classifier:\n",
        "    fit_params={'early_stopping_rounds':10,\\\n",
        "                         'eval_set':[(xvalid_count, valid_y_sw)]}\n",
        "    df_results = df_results.append(report(XGBClassifier(n_estimators=1000, subsample=0.8), xtrain_count,train_y_sw, name='XGB_Count_Vectors', cv=5, fit_params=fit_params, dict_scoring=score_metrics))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/joblib/externals/loky/process_executor.py:706: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
            "  \"timeout or by a memory leak.\", UserWarning\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 122 ms, sys: 366 ms, total: 488 ms\n",
            "Wall time: 5min 29s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hlftogroApb9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "10221fe1-cfca-4aee-bf51-3a3ecaacc74e"
      },
      "source": [
        "%%time\n",
        "if xgboost_classifier:\n",
        "    fit_params={'early_stopping_rounds':10,\\\n",
        "                         'eval_set':[(xvalid_tfidf, valid_y_sw)]}\n",
        "    df_results = df_results.append(report(XGBClassifier(n_estimators=1000, subsample=0.8), xtrain_tfidf,train_y, name='XGB_WordLevel_TF-IDF', cv=5, fit_params=fit_params, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 112 ms, sys: 41 ms, total: 153 ms\n",
            "Wall time: 6min 48s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_B85UH2dApcB",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "ee56d0e0-904e-43e7-83c0-fb30563d46c7"
      },
      "source": [
        "%%time\n",
        "if xgboost_classifier:\n",
        "    fit_params={'early_stopping_rounds':10,\\\n",
        "                         'eval_set':[(xvalid_tfidf_ngram, valid_y_sw)]}\n",
        "    df_results = df_results.append(report(XGBClassifier(n_estimators=1000, subsample=0.8), xtrain_tfidf_ngram,train_y, name='XGB_N-Gram_TF-IDF', cv=5, fit_params=fit_params, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 69.8 ms, sys: 16.1 ms, total: 85.9 ms\n",
            "Wall time: 50.4 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cy-ToR4dApcJ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 395
        },
        "outputId": "1d0179e1-21a4-40eb-e8e1-a6feb12d09af"
      },
      "source": [
        "%%time\n",
        "if xgboost_classifier:\n",
        "    fit_params={'early_stopping_rounds':10,\\\n",
        "                         'eval_set':[(xvalid_tfidf_ngram_chars, valid_y_sw)]}\n",
        "    df_results = df_results.append(report(XGBClassifier(n_estimators=1000, subsample=0.8), xtrain_tfidf_ngram_chars,train_y, name='XGB_CharLevel_TF-IDF', cv=5, fit_params=fit_params, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TerminatedWorkerError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTerminatedWorkerError\u001b[0m                     Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-47-86ba10e7f3c4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'time'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"if xgboost_classifier:\\n    fit_params={'early_stopping_rounds':10,\\\\\\n                         'eval_set':[(xvalid_tfidf_ngram_chars, valid_y_sw)]}\\n    df_results = df_results.append(report(XGBClassifier(n_estimators=1000, subsample=0.8), xtrain_tfidf_ngram_chars,train_y, name='XGB_CharLevel_TF-IDF', cv=5, fit_params=fit_params, dict_scoring=score_metrics))\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m   2115\u001b[0m             \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvar_expand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2116\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2117\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2118\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<decorator-gen-60>\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m         \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/IPython/core/magics/execution.py\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n\u001b[1;32m   1191\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1192\u001b[0m             \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1193\u001b[0;31m             \u001b[0mexec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglob\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_ns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1194\u001b[0m             \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32m<ipython-input-28-23dc97bec1c7>\u001b[0m in \u001b[0;36mreport\u001b[0;34m(clf, x, y, name, cv, dict_scoring, fit_params)\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;31m#if clf==XGBClassifier():\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     scores = cross_validate(clf, x, y, scoring=score,\n\u001b[0;32m---> 10\u001b[0;31m                          cv=cv, return_train_score=False, n_jobs=-1,  fit_params=fit_params)\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     71\u001b[0m                           FutureWarning)\n\u001b[1;32m     72\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36mcross_validate\u001b[0;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score, return_estimator, error_score)\u001b[0m\n\u001b[1;32m    246\u001b[0m             \u001b[0mreturn_times\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_estimator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_estimator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m             error_score=error_score)\n\u001b[0;32m--> 248\u001b[0;31m         for train, test in cv.split(X, y, groups))\n\u001b[0m\u001b[1;32m    249\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m     \u001b[0mzipped_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mscores\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1015\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1017\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1018\u001b[0m             \u001b[0;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1019\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    907\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    908\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'supports_timeout'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 909\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    910\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    911\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[0;34m(future, timeout)\u001b[0m\n\u001b[1;32m    560\u001b[0m         AsyncResults.get from multiprocessing.\"\"\"\n\u001b[1;32m    561\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 562\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    563\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mLokyTimeoutError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    564\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.6/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    430\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mCancelledError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    431\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mFINISHED\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 432\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    433\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    434\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.6/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    382\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    383\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 384\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    385\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTerminatedWorkerError\u001b[0m: A worker process managed by the executor was unexpectedly terminated. This could be caused by a segmentation fault while calling the function or by an excessive memory usage causing the Operating System to kill the worker. The exit codes of the workers are {SIGKILL(-9)}"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z0Zf2dvpApcO",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "61db1296-7960-493f-dab0-4bb0adad5558"
      },
      "source": [
        "%%time\n",
        "if xgboost_classifier:\n",
        "    fit_params={'early_stopping_rounds':10,\\\n",
        "                         'eval_set':[(valid_seq_x,valid_y)]}\n",
        "    df_results = df_results.append(report(XGBClassifier(n_estimators=1000, subsample=0.8), train_seq_x,train_y, name='XGB_Words', cv=5, fit_params=fit_params, dict_scoring=score_metrics))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 1.62 s, sys: 497 ms, total: 2.12 s\n",
            "Wall time: 2min 46s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P-7sNZAOApcS"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UGyDfd7oApcT"
      },
      "source": [
        "<center><h1>Deep Learning</h1></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ORhseQ19ApcU"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XADVh6JKApcV"
      },
      "source": [
        "<h3>Cohen’s kappa</h3>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oLMI3dvAApcW"
      },
      "source": [
        "The function [cohen_kappa_score](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.cohen_kappa_score.html#sklearn.metrics.cohen_kappa_score) computes [Cohen’s kappa](https://en.wikipedia.org/wiki/Cohen%27s_kappa) statistic. This measure is intended to compare labelings by different human annotators, not a classifier versus a ground truth.\n",
        "\n",
        "The kappa score (see docstring) is a number between -1 and 1. Scores above .8 are generally considered good agreement; zero or lower means no agreement (practically random labels).\n",
        "\n",
        "Kappa scores can be computed for binary or multiclass problems, but not for multilabel problems (except by manually computing a per-label score) and not for more than two annotators."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ji6XcKpiApcW"
      },
      "source": [
        "<h3>Balanced Accuracy</h3>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I1RZWqOhApcX"
      },
      "source": [
        "Compute the balanced accuracy\n",
        "\n",
        "The balanced accuracy in binary and multiclass classification problems to deal with imbalanced datasets. It is defined as the average of recall obtained on each class.\n",
        "\n",
        "The best value is 1 and the worst value is 0 when adjusted=False"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G98t6kOnApcY"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IIl2UTQ9ApcZ"
      },
      "source": [
        "<h3>Early Stopping, Model saving, Class weight configuration</h3>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pXnli6aRApcZ"
      },
      "source": [
        "es = tf.keras.callbacks.EarlyStopping(monitor='loss', mode='auto', patience=3)\n",
        "check_p = tf.keras.callbacks.ModelCheckpoint(\"save_models/model.h5\", save_best_only=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O1TOpBndApce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "outputId": "3376dfd7-0e5c-4f0b-81a7-08f36189202a"
      },
      "source": [
        "class_w = {}\n",
        "for i in zip(range(len(class_weights)), class_weights):\n",
        "    class_w[i[0]] = i[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-51-bf13c4c8c034>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mclass_w\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclass_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclass_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mclass_w\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'class_weights' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z4rGJVZiApci"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3jXMmMyJApck"
      },
      "source": [
        "def cross_validate_NN(model, X, y, X_test, y_test, callbacks,name=\"NN\", fit_params=None, scoring=None, n_splits=5):\n",
        "    #print(model.__class__.__name__)\n",
        "    # ---- Parameters initialisation\n",
        "    seed = 42\n",
        "    k = 1\n",
        "    np.random.seed(seed)\n",
        "    kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n",
        "\n",
        "    # Creation of list for each metric\n",
        "    if scoring==None:\n",
        "        dic_scoring = {}\n",
        "    if scoring!=None:\n",
        "        dic_score = scoring.copy()\n",
        "\n",
        "    dic_score[\"fit_time\"] = None\n",
        "    dic_score[\"score_time\"] = None\n",
        "    scorer = {}\n",
        "    for i in dic_score.keys():\n",
        "        scorer[i] = []\n",
        "\n",
        "\n",
        "    index = [\"Model\"]\n",
        "    results = [name]\n",
        "    # ---- Loop on k-fold for cross-valisation\n",
        "    for train, test in kfold.split(X, y):\n",
        "        # create model\n",
        "        print(f\"k-fold : {k}\")\n",
        "        fit_start = time.time()\n",
        "        _model = model\n",
        "        _model.fit(X[train], y[train],\n",
        "                        epochs=1000, callbacks=[callbacks],\n",
        "                        validation_split=0.2, verbose=False)\n",
        "\n",
        "        fit_end = time.time() - fit_start\n",
        "\n",
        "        _acc = _model.evaluate(X_test, y_test, verbose=0)\n",
        "\n",
        "        score_start = time.time()\n",
        "        y_pred = (model.predict(X_test)>0.5).astype(int)\n",
        "        score_end = time.time() - score_start\n",
        "\n",
        "        # ---- save each metric\n",
        "        for i in dic_score.keys():\n",
        "            if i == \"fit_time\":\n",
        "                scorer[i].append(fit_end)\n",
        "                index.append(i+'_cv'+str(k))\n",
        "                results.append(fit_end)\n",
        "                continue\n",
        "            if i == \"score_time\":\n",
        "                scorer[i].append(score_end)\n",
        "                index.append(i+'_cv'+str(k))\n",
        "                results.append(score_end)\n",
        "                continue\n",
        "\n",
        "            scorer[i].append(dic_score[i](y_test, y_pred))\n",
        "            index.append(\"test_\"+i+'_cv'+str(k))\n",
        "            results.append(scorer[i][-1])\n",
        "\n",
        "\n",
        "        k+=1\n",
        "\n",
        "    # Compute mean and std for each metric\n",
        "    for i in scorer:\n",
        "\n",
        "        results.append(np.mean(scorer[i]))\n",
        "        results.append(np.std(scorer[i]))\n",
        "        if i == \"fit_time\":\n",
        "            index.append(i+\"_mean\")\n",
        "            index.append(i+\"_std\")\n",
        "            continue\n",
        "        if i == \"score_time\":\n",
        "            index.append(i+\"_mean\")\n",
        "            index.append(i+\"_std\")\n",
        "            continue\n",
        "\n",
        "        index.append(\"test_\"+i+\"_mean\")\n",
        "        index.append(\"test_\"+i+\"_std\")\n",
        "\n",
        "\n",
        "\n",
        "    return pd.DataFrame(results, index=index).T"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AiK-9BpDApco"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZYfWlV5FApcp"
      },
      "source": [
        "<h2>Shallow Neural Networks</h2>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gKjxpwj8kkJP"
      },
      "source": [
        "from tensorflow import keras"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nSMDB4AdApcq"
      },
      "source": [
        "def shallow_neural_networks(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 16)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "      embedded,\n",
        "      keras.layers.GlobalAveragePooling1D(),\n",
        "\n",
        "      keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MiIIf4j6Apcu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "aa88581c-afdf-4cec-964d-c3873116c021"
      },
      "source": [
        "%%time\n",
        "if shallow_network:\n",
        "    df_results = df_results.append(cross_validate_NN(shallow_neural_networks(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"Shallow_NN_WE\", scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "k-fold : 1\n",
            "k-fold : 2\n",
            "k-fold : 3\n",
            "k-fold : 4\n",
            "k-fold : 5\n",
            "CPU times: user 1h 40min 46s, sys: 3min 52s, total: 1h 44min 38s\n",
            "Wall time: 57min 5s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YIPUc53PApcy"
      },
      "source": [
        "#c = cross_validate_NN(shallow_neural_networks(word_index, pre_trained=pre_trained), train_seq_x[:100], train_y[:100], valid_seq_x[:100], valid_y[:100], es, name=\"Shallow_NN_WE\", scoring=score_metrics, n_splits=2)\n",
        "#c[[\"test_acc_cv1\", \"test_recall_cv1\"]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "09ASb-5WApc2"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uW9kVQfiApc3"
      },
      "source": [
        "<h2>Deep Neural Networks</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kJipI2FrApc9"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uP8eL0asApc-"
      },
      "source": [
        "def deep_neural_networks(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 50)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "      embedded,\n",
        "      keras.layers.GlobalAveragePooling1D(),\n",
        "      keras.layers.Dense(16, activation='relu'),\n",
        "      keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "psMJGu-AApdD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "e0dea10a-d13c-460d-f9fb-473d44d76a55"
      },
      "source": [
        "%%time\n",
        "if deep_nn:\n",
        "    df_results = df_results.append(cross_validate_NN(deep_neural_networks(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"Deep_NN_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "k-fold : 1\n",
            "k-fold : 2\n",
            "k-fold : 3\n",
            "k-fold : 4\n",
            "k-fold : 5\n",
            "CPU times: user 3h 12min 11s, sys: 7min 2s, total: 3h 19min 14s\n",
            "Wall time: 1h 45min 50s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XqZYySZlApdI"
      },
      "source": [
        "<h2>Deep Neural Networks variation 1</h2>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qmVBSsJWApdJ"
      },
      "source": [
        "def deep_neural_networks_var1(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "      embedded,\n",
        "      keras.layers.GlobalAveragePooling1D(),\n",
        "      keras.layers.Dense(16, activation='relu'),\n",
        "      keras.layers.Dense(16, activation='relu'),\n",
        "      keras.layers.Dense(1  if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "baFbYpxUApdO",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "0d201c90-a856-43f2-f5a3-b4d29e347839"
      },
      "source": [
        "%%time\n",
        "if deep_nn:\n",
        "    df_results = df_results.append(cross_validate_NN(deep_neural_networks_var1(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"Deep_NN_var1_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "k-fold : 1\n",
            "k-fold : 2\n",
            "k-fold : 3\n",
            "k-fold : 4\n",
            "k-fold : 5\n",
            "CPU times: user 4h 44min 18s, sys: 8min 26s, total: 4h 52min 45s\n",
            "Wall time: 2h 34min 15s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-dp2cRzCApdU"
      },
      "source": [
        "<h2>Deep Neural Networks variation 2</h2>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FpUiWlEuApdV"
      },
      "source": [
        "def deep_neural_networks_var2(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "      embedded,\n",
        "      keras.layers.GlobalAveragePooling1D(),\n",
        "      keras.layers.Dense(32, activation='relu'),\n",
        "      keras.layers.Dense(16, activation='relu'),\n",
        "      keras.layers.Dense(1  if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GZ21tAlKApdZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "8fa0d4d5-0e8a-416b-b55e-b4685b39e951"
      },
      "source": [
        "%%time\n",
        "if deep_nn:\n",
        "    df_results = df_results.append(cross_validate_NN(deep_neural_networks_var2(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"Deep_NN_var2_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "k-fold : 1\n",
            "k-fold : 2\n",
            "k-fold : 3\n",
            "k-fold : 4\n",
            "k-fold : 5\n",
            "CPU times: user 4h 3min 36s, sys: 5min 37s, total: 4h 9min 14s\n",
            "Wall time: 2h 9min 28s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e_mDQKiqApdd"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "354n9ydZApde"
      },
      "source": [
        "<h2>Recurent Neural Network - RNN</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ee4JuESApdf"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7AGTwCclApdg"
      },
      "source": [
        "def create_rnn_model(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.SimpleRNN(40, return_sequences=True),\n",
        "    keras.layers.SimpleRNN(40, return_sequences=True),\n",
        "    keras.layers.SimpleRNN(40, return_sequences=True),\n",
        "    keras.layers.SimpleRNN(40),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=1e-4),\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KG8KmIXSApdj",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6fcad4a7-9fc3-4485-c328-fb62dae7ab8d"
      },
      "source": [
        "%%time\n",
        "if rnn:\n",
        "    df_results = df_results.append(cross_validate_NN(create_rnn_model(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"RNN_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "k-fold : 1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UoAep_ERApds"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WgFMLmjaApdt"
      },
      "source": [
        "<h2>Convolutional Neural Network</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Clsm-kLvApdu"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H4Jw4ulMApdv"
      },
      "source": [
        "def create_conv_model(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) +1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.Conv1D(100, 5, activation='relu'), # padding='same'\n",
        "    keras.layers.Dropout(0.2),\n",
        "    keras.layers.MaxPooling1D(pool_size=4),\n",
        "    keras.layers.Conv1D(64, 5, activation='relu'),\n",
        "    keras.layers.Dropout(0.2),\n",
        "    keras.layers.MaxPooling1D(pool_size=4),\n",
        "    keras.layers.Conv1D(32, 5, activation='relu'),\n",
        "    keras.layers.Dropout(0.2),\n",
        "    keras.layers.GlobalMaxPooling1D(),\n",
        "\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=1e-4),\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VhSdMGQMApd0"
      },
      "source": [
        "%%time\n",
        "if cnn:\n",
        "    df_results = df_results.append(cross_validate_NN(create_rnn_model(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"CNN_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qhcazt9iApd9"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vQDyLQzUApd-"
      },
      "source": [
        "<h2>Recurrent Neural Network – LSTM</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uCOlRMxlApd_"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y_Ccbz-ZApeA"
      },
      "source": [
        "def create_lstm_model(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) +1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index)+1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.LSTM(32),\n",
        "    keras.layers.Dropout(0.2),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "74rZBJ0jApeF"
      },
      "source": [
        "%%time\n",
        "if lstm:\n",
        "    df_results = df_results.append(cross_validate_NN(create_lstm_model(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"LSTM_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rvGqmIxIApeJ"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "poS1fmnAApeK"
      },
      "source": [
        "<h2>CNN – LSTM</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6fYqaH9sApeK"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e1i3wxzYApeL"
      },
      "source": [
        "def create_cnn_lstm_model(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.Conv1D(128, 5, activation='relu'),\n",
        "    keras.layers.Dropout(0.2),\n",
        "    keras.layers.MaxPooling1D(pool_size=4),\n",
        "    keras.layers.LSTM(32),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bnmKt2CjApeQ"
      },
      "source": [
        "%%time\n",
        "if cnn_lstm:\n",
        "    df_results = df_results.append(cross_validate_NN(create_cnn_lstm_model(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es,name=\"CNN_LSTM_WE\", scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3YRo-hd2ApeT"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DmDvA_tFApeU"
      },
      "source": [
        "<h2>CNN – GRU</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F-M3mpMxApeV"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6fdqGPURApeV"
      },
      "source": [
        "def create_cnn_gru_model(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.Conv1D(128, 5, activation='relu'),\n",
        "    keras.layers.Dropout(0.2),\n",
        "    keras.layers.MaxPooling1D(pool_size=4),\n",
        "    keras.layers.GRU(32),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-jD2OFTJApeb"
      },
      "source": [
        "%%time\n",
        "if cnn_gru:\n",
        "    df_results = df_results.append(cross_validate_NN(create_cnn_gru_model(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"CNN_GRU_WE\", scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eWUyR-QmApee"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aLAwfS3BApef"
      },
      "source": [
        "<h2>Recurrent Neural Network – GRU</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_56NrbUkApeg"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IBLDE6VdApeh"
      },
      "source": [
        "tf.keras.layers.GRU(\n",
        "    units, activation='tanh', recurrent_activation='sigmoid', use_bias=True,\n",
        "    kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal',\n",
        "    bias_initializer='zeros', kernel_regularizer=None, recurrent_regularizer=None,\n",
        "    bias_regularizer=None, activity_regularizer=None, kernel_constraint=None,\n",
        "    recurrent_constraint=None, bias_constraint=None, dropout=0.0,\n",
        "    recurrent_dropout=0.0, implementation=2, return_sequences=False,\n",
        "    return_state=False, go_backwards=False, stateful=False, unroll=False,\n",
        "    time_major=False, reset_after=True, **kwargs\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4dOK7Ce8Apei"
      },
      "source": [
        "def create_gru_model(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.GRU(32),\n",
        "    keras.layers.Dropout(0.2),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7iFeOvFlApem"
      },
      "source": [
        "%%time\n",
        "if gru:\n",
        "    df_results = df_results.append(cross_validate_NN(create_gru_model(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"GRU_WE\", scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r79vIyYWApeq"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T4tanJBoAper"
      },
      "source": [
        "<h2>Bidirectional RNN</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gfYJR8cCApes"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "emrtTdrCApet"
      },
      "source": [
        "def create_bidirec_rnn_model(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.Bidirectional(keras.layers.SimpleRNN(32, return_sequences=True)),\n",
        "    keras.layers.Bidirectional(keras.layers.SimpleRNN(32, return_sequences=True)),\n",
        "    keras.layers.Bidirectional(keras.layers.SimpleRNN(32, return_sequences=True)),\n",
        "    keras.layers.Bidirectional(keras.layers.SimpleRNN(32)),\n",
        "    keras.layers.Dropout(0.2),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-nH4JlYjApe2"
      },
      "source": [
        "%%time\n",
        "if bidirectional_rnn:\n",
        "    df_results = df_results.append(cross_validate_NN(create_bidirec_rnn_model(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"BiRNN_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GWdgekCxApe6"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xiaJwa0WApe7"
      },
      "source": [
        "<h2>Bidirectional LSTM</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "japIsjbAApe7"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nltXK10HApe8"
      },
      "source": [
        "def create_bidirec_lstm_model(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.Bidirectional(keras.layers.LSTM(32)),\n",
        "    keras.layers.Dropout(0.2),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5SWMc4t7Ape_"
      },
      "source": [
        "%%time\n",
        "if bidirectional_lstm:\n",
        "    df_results = df_results.append(cross_validate_NN(create_bidirec_lstm_model(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"BiLSTM_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hcZCbf8XApfC"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d6r-HJUcApfD"
      },
      "source": [
        "<h2>Bidirectional GRU</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CdWcmCLzApfE"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ax1n71-YApfF"
      },
      "source": [
        "def create_bidirec_gru_model(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.Bidirectional(keras.layers.GRU(32)),\n",
        "    keras.layers.Dropout(0.2),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TEIeZwC5ApfI"
      },
      "source": [
        "%%time\n",
        "if bidirectional_gru:\n",
        "    df_results = df_results.append(cross_validate_NN(create_bidirec_gru_model(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"BiGRU_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bwP8NIsmApfM"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EUUeKRH3ApfM"
      },
      "source": [
        "<h2>Recurrent Convolutional Neural Network</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n87gjFu7ApfN"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iUP4i-TYApfO"
      },
      "source": [
        "def create_rcnn(X, word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300,input_length=X.shape[1], weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.SpatialDropout1D(0.3),\n",
        "    keras.layers.Bidirectional(keras.layers.GRU(32,return_sequences=True)),\n",
        "    keras.layers.Convolution1D(32, 3, activation=\"relu\"),\n",
        "    keras.layers.GlobalMaxPool1D(),\n",
        "    keras.layers.Dense(25, activation=\"relu\"),\n",
        "    keras.layers.Dropout(0.25),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9B3vH3W5ApfR"
      },
      "source": [
        "%%time\n",
        "if rcnn:\n",
        "    df_results = df_results.append(cross_validate_NN(create_rcnn(train_seq_x, word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"RCNN_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6HhDeMJdApfU"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FsY_e9M3ApfV"
      },
      "source": [
        "<h2>Recurrent Convolutional Neural Network variation 1</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lAUlT-fBApfX"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mfaYy4J3ApfY"
      },
      "source": [
        "def create_rcnn_var1(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.SpatialDropout1D(0.3),\n",
        "    keras.layers.Bidirectional(keras.layers.LSTM(32,return_sequences=True)),\n",
        "    keras.layers.Convolution1D(32, 3, activation=\"relu\"),\n",
        "    keras.layers.GlobalMaxPool1D(),\n",
        "    keras.layers.Dense(25, activation=\"relu\"),\n",
        "    keras.layers.Dropout(0.25),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mc_QxJC_Apfb"
      },
      "source": [
        "%%time\n",
        "if rcnn:\n",
        "    df_results = df_results.append(cross_validate_NN(create_rcnn_var1(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"RCNN_var1_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VrKZrRqHApff"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m4MH0OEpApff"
      },
      "source": [
        "<h2>Recurrent Convulational Neural Network variation 2</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dPsZfeyMApfg"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZynuyPsNApfh"
      },
      "source": [
        "def create_rcnn_var2(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.SpatialDropout1D(0.3),\n",
        "    keras.layers.Bidirectional(keras.layers.GRU(32,return_sequences=True)),\n",
        "    keras.layers.Bidirectional(keras.layers.GRU(32,return_sequences=True)),\n",
        "    keras.layers.Convolution1D(32, 3, activation=\"relu\"),\n",
        "    keras.layers.GlobalMaxPool1D(),\n",
        "    keras.layers.Dense(25, activation=\"relu\"),\n",
        "    keras.layers.Dropout(0.25),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qlJkO1puApfk"
      },
      "source": [
        "%%time\n",
        "if rcnn:\n",
        "    df_results = df_results.append(cross_validate_NN(create_rcnn_var2(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"RCNN_var2_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jJ5vzqX3Apfo"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4GtBqbM_Apfp"
      },
      "source": [
        "<h2>Recurrent Convulational Neural Network variation 3</h2>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LlLyzxcsApfp"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YMph28lOApfu"
      },
      "source": [
        "def create_rcnn_var3(word_index, label=labels, embedding_matrix=embedding_matrix, pre_trained=False):\n",
        "    if pre_trained==False:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 100)\n",
        "    else:\n",
        "        embedded = keras.layers.Embedding(len(word_index) + 1, 300, weights=[embedding_matrix], trainable=False)\n",
        "\n",
        "    model = keras.Sequential([\n",
        "    embedded,\n",
        "    keras.layers.SpatialDropout1D(0.3),\n",
        "    keras.layers.Bidirectional(keras.layers.GRU(32,return_sequences=True)),\n",
        "    keras.layers.Bidirectional(keras.layers.LSTM(32,return_sequences=True)),\n",
        "    keras.layers.Convolution1D(32, 3, activation=\"relu\"),\n",
        "    keras.layers.GlobalMaxPool1D(),\n",
        "    keras.layers.Dense(25, activation=\"relu\"),\n",
        "    keras.layers.Dropout(0.25),\n",
        "    keras.layers.Dense(1 if len(label)<=2 else len(label), activation='sigmoid' if len(label)<=2 else \"softmax\")])\n",
        "\n",
        "    if len(label)==2:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    else:\n",
        "        model.compile(optimizer='adam',\n",
        "              loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])\n",
        "    #print(model.summary())\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xvF5sviFApf4"
      },
      "source": [
        "%%time\n",
        "if rcnn:\n",
        "    df_results = df_results.append(cross_validate_NN(create_rcnn_var3(word_index, pre_trained=pre_trained), train_seq_x, train_y, valid_seq_x, valid_y, es, name=\"RCNN_var3_WE\",scoring=score_metrics, n_splits=5))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JtgkBj7GApf9"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zXjtAx2dApf-"
      },
      "source": [
        "<center><h1>Results</h1></center>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ikXG5wuTApf-"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BRH4xvQOApf_"
      },
      "source": [
        "df_results = df_results.reset_index()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dcx12ZwuApgC"
      },
      "source": [
        "df_results[[ \"Model\",\"test_acc_mean\",\"test_acc_std\",\n",
        "                        \"test_balanced_accuracy_mean\",\"test_balanced_accuracy_std\",\n",
        "                       \"test_prec_mean\", \"test_prec_std\",\n",
        "                        \"test_recall_mean\",\"test_recall_std\",\n",
        "                       \"test_f1-score_mean\", \"test_f1-score_std\",\n",
        "                       \"test_cohens_kappa_mean\", \"test_cohens_kappa_std\", \"test_matthews_corrcoef_mean\",\"test_matthews_corrcoef_std\",\n",
        "                       \"test_roc_auc_mean\", \"test_roc_auc_std\"]][df_results[\"test_prec_mean\"]<1].sort_values(by=[\"test_prec_mean\", \"test_recall_mean\"], ascending=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dKY9PvOFApgF"
      },
      "source": [
        "if save_results:\n",
        "    df_results.sort_values(by=[\"test_prec_mean\", \"test_recall_mean\"], ascending=False).to_csv(\"model_selection_results_mails.csv\", index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HME9hFWLApgI"
      },
      "source": [],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LJ6yLObzApgL"
      },
      "source": [
        "for i in df_results.iloc[35]: print(i)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n48dCFQrApgO"
      },
      "source": [],
      "execution_count": null,
      "outputs": []
    }
  ]
}