Federated_Detection_Of_Hate_Speech.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pxx1EnSbnNnN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        },
        "outputId": "0e7045ba-5047-4442-999c-d3ebfb784856"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\nMIT License\\n\\nCopyright (c) 2021 Valentin Gorbunov\\n\\nPermission is hereby granted, free of charge, to any person obtaining a copy\\nof this software and associated documentation files (the \"Software\"), to deal\\nin the Software without restriction, including without limitation the rights\\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\\ncopies of the Software, and to permit persons to whom the Software is\\nfurnished to do so, subject to the following conditions:\\n\\nThe above copyright notice and this permission notice shall be included in all\\ncopies or substantial portions of the Software.\\n\\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\\nSOFTWARE.\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "'''\n",
        "MIT License\n",
        "\n",
        "Copyright (c) 2021 Valentin Gorbunov\n",
        "\n",
        "Permission is hereby granted, free of charge, to any person obtaining a copy\n",
        "of this software and associated documentation files (the \"Software\"), to deal\n",
        "in the Software without restriction, including without limitation the rights\n",
        "to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
        "copies of the Software, and to permit persons to whom the Software is\n",
        "furnished to do so, subject to the following conditions:\n",
        "\n",
        "The above copyright notice and this permission notice shall be included in all\n",
        "copies or substantial portions of the Software.\n",
        "\n",
        "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
        "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
        "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n",
        "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
        "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n",
        "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n",
        "SOFTWARE.\n",
        "'''"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hH_AXG4tBXY9"
      },
      "source": [
        "## Environment Set Up"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "06McKulLLqQf",
        "outputId": "384f5534-963e-442d-de56-414e903c2eec"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: vaderSentiment in /usr/local/lib/python3.7/dist-packages (3.3.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from vaderSentiment) (2.23.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->vaderSentiment) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->vaderSentiment) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->vaderSentiment) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->vaderSentiment) (3.0.4)\n",
            "Requirement already satisfied: textstat in /usr/local/lib/python3.7/dist-packages (0.7.3)\n",
            "Requirement already satisfied: pyphen in /usr/local/lib/python3.7/dist-packages (from textstat) (0.12.0)\n",
            "Requirement already satisfied: flwr in /usr/local/lib/python3.7/dist-packages (0.18.0)\n",
            "Requirement already satisfied: importlib-metadata<2.0.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from flwr) (1.7.0)\n",
            "Requirement already satisfied: numpy<2.0.0,>=1.19.0 in /usr/local/lib/python3.7/dist-packages (from flwr) (1.21.6)\n",
            "Requirement already satisfied: grpcio<=1.43.0,>=1.27.2 in /usr/local/lib/python3.7/dist-packages (from flwr) (1.43.0)\n",
            "Requirement already satisfied: protobuf<4.0.0,>=3.12.1 in /usr/local/lib/python3.7/dist-packages (from flwr) (3.17.3)\n",
            "Requirement already satisfied: google<3.0.0,>=2.0.3 in /usr/local/lib/python3.7/dist-packages (from flwr) (2.0.3)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from google<3.0.0,>=2.0.3->flwr) (4.6.3)\n",
            "Requirement already satisfied: six>=1.5.2 in /usr/local/lib/python3.7/dist-packages (from grpcio<=1.43.0,>=1.27.2->flwr) (1.15.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata<2.0.0,>=1.4.0->flwr) (3.8.0)\n"
          ]
        }
      ],
      "source": [
        "# Install packages not in Colab by default\n",
        "!pip install vaderSentiment\n",
        "!pip install textstat\n",
        "!pip install flwr"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mMKXiA0eLLOG"
      },
      "outputs": [],
      "source": [
        "# Import packages\n",
        "import pickle\n",
        "import sys\n",
        "import string\n",
        "import re\n",
        "import nltk\n",
        "import seaborn\n",
        "import os\n",
        "import time\n",
        "import warnings\n",
        "import random\n",
        "import threading\n",
        "import scipy\n",
        "import json\n",
        "import nltk.stem.porter\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import flwr as fl\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib import rcParams\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.feature_selection import SelectFromModel\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.metrics import classification_report, confusion_matrix, log_loss, precision_recall_fscore_support\n",
        "from sklearn.model_selection import train_test_split, StratifiedKFold\n",
        "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer as VS\n",
        "from textstat.textstat import *\n",
        "from typing import Tuple, Union, List, Dict, Optional\n",
        "from flwr.server.criterion import Criterion\n",
        "from flwr.server.client_proxy import ClientProxy\n",
        "from flwr.common import Weights, FitRes, Parameters, Scalar, parameters_to_weights, weights_to_parameters\n",
        "from google.colab import drive\n",
        "from multiprocessing import Process, Manager\n",
        "from multiprocessing.managers import ListProxy\n",
        "from math import ceil\n",
        "from statistics import mean\n",
        "from datetime import datetime\n",
        "from functools import reduce\n",
        "import textwrap\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yvAKAU9TLgkO",
        "outputId": "aac7a78d-42d9-445d-859d-5ce1dcc5aa02"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "# Mount external storage\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v_0ZjZ5JGptA"
      },
      "outputs": [],
      "source": [
        "# Configure global search paths\n",
        "HOME_DIR = \"/content/drive/MyDrive/\"\n",
        "LOGGING_FOLDERS_DIR = HOME_DIR + \"Results_Folders/\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zQMWbh0ogBG0"
      },
      "source": [
        "## Centralized Manual Pipeline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "id": "-35WzAX-gc5b",
        "outputId": "37df8647-3a3b-47ad-b0be-421eeb336e52"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\nAdapted from:\\n\\nTitle: Automated Hate Speech Detection and the Problem of Offensive Language.ipynb\\nAuthors: Davidson et al.\\nDate: Mar 29, 2019\\nAvailability: https://github.com/t-davidson/hate-speech-and-offensive-language/blob/master/src/Automated%20Hate%20Speech%20Detection%20and%20the%20Problem%20of%20Offensive%20Language.ipynb\\n\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "'''\n",
        "Centralized Manual Pipeline Adapted From:\n",
        "\n",
        "Title: Automated Hate Speech Detection and the Problem of Offensive Language.ipynb\n",
        "Authors: Davidson et al.\n",
        "Date: Mar 29, 2019\n",
        "Availability: https://github.com/t-davidson/hate-speech-and-offensive-language/blob/master/src/Automated%20Hate%20Speech%20Detection%20and%20the%20Problem%20of%20Offensive%20Language.ipynb\n",
        "\n",
        "'''"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hwKkn2DTLLOH"
      },
      "source": [
        "### Preliminary Data Analysis"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xpVnw4tzLLOI"
      },
      "outputs": [],
      "source": [
        "# Deserialize Pandas DataFrame from pickle file\n",
        "df = pd.read_pickle(open(HOME_DIR + \"labeled_data.p\", mode='rb'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "tCg6WZzCLLOI",
        "outputId": "18e9467a-e6b7-483d-e9b6-cd23885f7d28"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       count  hate_speech  offensive_language  neither class  \\\n",
              "0          3            0                   0        3     2   \n",
              "1          3            0                   3        0     1   \n",
              "2          3            0                   3        0     1   \n",
              "3          3            0                   2        1     1   \n",
              "4          6            0                   6        0     1   \n",
              "...      ...          ...                 ...      ...   ...   \n",
              "25291      3            0                   2        1     1   \n",
              "25292      3            0                   1        2     2   \n",
              "25294      3            0                   3        0     1   \n",
              "25295      6            0                   6        0     1   \n",
              "25296      3            0                   0        3     2   \n",
              "\n",
              "                                                   tweet  \n",
              "0      !!! RT @mayasolovely: As a woman you shouldn't...  \n",
              "1      !!!!! RT @mleew17: boy dats cold...tyga dwn ba...  \n",
              "2      !!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...  \n",
              "3      !!!!!!!!! RT @C_G_Anderson: @viva_based she lo...  \n",
              "4      !!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...  \n",
              "...                                                  ...  \n",
              "25291  you's a muthaf***in lie &#8220;@LifeAsKing: @2...  \n",
              "25292  you've gone and broke the wrong heart baby, an...  \n",
              "25294  young buck wanna eat!!.. dat nigguh like I ain...  \n",
              "25295              youu got wild bitches tellin you lies  \n",
              "25296  ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...  \n",
              "\n",
              "[24783 rows x 6 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8c6a7709-226d-4af6-ae5e-23e518398cb2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "      <th>hate_speech</th>\n",
              "      <th>offensive_language</th>\n",
              "      <th>neither</th>\n",
              "      <th>class</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>!!! RT @mayasolovely: As a woman you shouldn't...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>!!!!! RT @mleew17: boy dats cold...tyga dwn ba...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>!!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>!!!!!!!!! RT @C_G_Anderson: @viva_based she lo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>!!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25291</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>you's a muthaf***in lie &amp;#8220;@LifeAsKing: @2...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25292</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25294</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>young buck wanna eat!!.. dat nigguh like I ain...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25295</th>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>youu got wild bitches tellin you lies</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25296</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>~~Ruffled | Ntac Eileen Dahlia - Beautiful col...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>24783 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8c6a7709-226d-4af6-ae5e-23e518398cb2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8c6a7709-226d-4af6-ae5e-23e518398cb2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8c6a7709-226d-4af6-ae5e-23e518398cb2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "# Describe content, size and format of the data\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "EzxqvWu-LLOJ",
        "outputId": "907fc6ff-1f84-4f2e-bba8-66da986166d5"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "              count   hate_speech  offensive_language       neither\n",
              "count  24783.000000  24783.000000        24783.000000  24783.000000\n",
              "mean       3.243473      0.280515            2.413711      0.549247\n",
              "std        0.883060      0.631851            1.399459      1.113299\n",
              "min        3.000000      0.000000            0.000000      0.000000\n",
              "25%        3.000000      0.000000            2.000000      0.000000\n",
              "50%        3.000000      0.000000            3.000000      0.000000\n",
              "75%        3.000000      0.000000            3.000000      0.000000\n",
              "max        9.000000      7.000000            9.000000      9.000000"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a7bc4a59-d993-4306-b6b2-dc2cef808552\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "      <th>hate_speech</th>\n",
              "      <th>offensive_language</th>\n",
              "      <th>neither</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>24783.000000</td>\n",
              "      <td>24783.000000</td>\n",
              "      <td>24783.000000</td>\n",
              "      <td>24783.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>3.243473</td>\n",
              "      <td>0.280515</td>\n",
              "      <td>2.413711</td>\n",
              "      <td>0.549247</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.883060</td>\n",
              "      <td>0.631851</td>\n",
              "      <td>1.399459</td>\n",
              "      <td>1.113299</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>3.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>3.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>3.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>9.000000</td>\n",
              "      <td>7.000000</td>\n",
              "      <td>9.000000</td>\n",
              "      <td>9.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a7bc4a59-d993-4306-b6b2-dc2cef808552')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a7bc4a59-d993-4306-b6b2-dc2cef808552 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a7bc4a59-d993-4306-b6b2-dc2cef808552');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "# Generate descriptive statistics\n",
        "df.describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "POVCiaOcLLOK",
        "outputId": "f276af6e-2590-4477-ab7c-3101606284db"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['count', 'hate_speech', 'offensive_language', 'neither', 'class',\n",
              "       'tweet'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "# Get feature names \n",
        "df.columns"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rMFJw6U9LLOL"
      },
      "source": [
        "Key:\n",
        "\n",
        "count = number of CrowdFlower users who coded each tweet (at least 3; more users coded a tweet when judgments were determined to be unreliable by CF)\n",
        "\n",
        "\n",
        "hate_speech = number of CF users who coded the tweet as hate speech\n",
        "\n",
        "\n",
        "offensive_language = number of CF users who coded the tweet as offensive but not hate speech\n",
        "\n",
        "\n",
        "neither = number of CF users who coded the tweet as neither offensive nor hate speech\n",
        "\n",
        "\n",
        "class = class label determined by majority rule:\n",
        "\n",
        "* 0 : hate speech\n",
        "* 1 : offensive  language\n",
        "* 2 : neither\n",
        "\n",
        "tweet = tweet text\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "lW-mzJD40pUk",
        "outputId": "08b6da08-1492-4288-cf1d-6915bb025d66"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEWCAYAAACnlKo3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5wcVZ338c+XcL9JIGMMCRguAQWEACPgBY0iEEFJcBXhEQnIQ+BBZFllBV1XWJVdXAU0XsCg2QRUwh0iBGNAAS9AMoFAEhATIG4mhDAQIFwDgd/zxzktxdAz6VSmuzPM9/169WuqfnWq6lT3zPz6nFN9WhGBmZlZGes0uwJmZtZ7OYmYmVlpTiJmZlaak4iZmZXmJGJmZqU5iZiZWWlOImZNImlnSbMlPSvp1BL7HyvpT/WoW0+SdKuk/9vofa0xnESsLiQtlPSipOcKj62bXa+1zFeBP0TEZhExrloBSQdLuj0nmg5Jt0k6rMH1rNRloaSPNePctvZyErF6+mREbFp4PFrcKGndZlVsLfFOYF5XGyV9GrgSuAQYAgwEvgl8siG1M6uBk4g1lKSQ9EVJ84H5OfaJ3K3ztKS/SNq9UH5PSXfnd+KXS5os6Tt525u6c/Lxd8zLG0j6vqT/lbRU0kWSNsrbRkhql/QVSY9LWiLpuMJxNpJ0nqS/S3pG0p9y7EZJX+p0zvskHd7F9R4maV6+tlslvTvHfw98BPhxbqXt1Gk/AecD346In0fEMxHxWkTcFhEndHGuH0paJGm5pFmS9i9s20dSW962VNL5Ob6hpF9KejLXcaakgd2+iG8+b39JN+SW0lN5eUinYjtImpHPf72kLQv775df96cl3StpRBfn2TG3xJ6R9ISky1ennlYfTiLWDKOBfYFdJO0JTABOBLYCfgZMyQlgfeA64FJgS9K78n9ajfOcC+wEDAd2BAaT3slXvAN4W44fD/xEUv+87fvA3sD787m/CrwGTAKOrhxA0h55/xs7nzwnhsuA04AWYCrwG0nrR8RHgT8Cp+RW2t867b4zsA1w1Wpc78x8rVsCvwaulLRh3vZD4IcRsTmwA3BFjo/Jz8E2pOf/JODF1TgnpP8j/0NqWW2b9/9xpzLHAF8ABgErgXEAkirP3XdyvU8HrpbUUuU83wZ+B/Qntcx+tJr1tDpwErF6ui6/u3xa0nWF+H9FxLKIeBEYC/wsIu6KiFcjYhKwAtgvP9YDfhARr0TEVaR/lKuU38mPBf4ln+tZ4D+BIwvFXgG+lY89FXgO2FnSOqR/eP8cEYtzvf4SESuAKcBOkoblY3weuDwiXq5Sjc8CN0bE9Ih4hZSYNiIlplXZKv9cUsv1AkTELyPiyYhYGRHnARuQklHlWneUNCAinouIOwvxrYAd83XOiojltZ4zn/fJiLg6Il7Iz/M5wIc7Fbs0IuZGxPPAvwNHSOpHSshTI2JqbmlNB9qAQ6qc6hVSoto6Il6KiLX+poK+wEnE6ml0RGyRH6ML8UWF5XcCXykkm6dJ74q3zo/F8cZZQv9e47lbgI2BWYXj/jbHK56MiJWF9ReATYEBwIbAQ50PGhEvAZcDR+dkcxSppVTN1sX6RsRrpGsfXEP9n8w/B9VQFgBJp0t6IHf3PE1qYQzIm48ntcr+mrusPpHjlwLTgMmSHpX035LWq/Wc+bwbS/pZ7vpbDtwObJGTREXxNf876c3BANLr/5lOr/8Hu7jurwICZuQuwi+sTj2tPpxErBmKSWERcE4h2WwRERtHxGWkd+GDc6uiYtvC8vOkRAGApHcUtj1B6lbZtXDct0XEpjXU7wngJVK3TzWTgM8BBwAvRMQdXZR7lPRPslI/kRLk4hrq8CDpuamp+y6Pf3wVOALoHxFbAM+Q/ukSEfMj4ijg7cB3gaskbZJbYf8REbuQWkifIHU9rY6vkFo8++busg9VqlUos01heVtSq+KJfI2Xdnr9N4mIczufJCIei4gTImJrUvfnTyvjX9Y8TiLWbBcDJ0naV8kmkg6VtBlwB6n//FRJ60n6FLBPYd97gV0lDc99/2dXNuR3/RcDF0h6O6T+d0kHr6pCed8JwPmStpbUT9L7JG2Qt99BGh85j65bIZDGHQ6VdEB+d/8VUlfdX2qoQwBfBv5d0nGSNpe0jqQPShpfZZfNSM9VB7CupG8Cm1c2SjpaUku+tqdz+DVJH5H0ntxqWE765/5aN1VbLw/GVx7r5nO/CDydB8zPqrLf0ZJ2kbQx8C3gqoh4Ffgl8EmlW5n75WOOqDIwj6TPFOJPkd6MdFdXawAnEWuqiGgDTiANxD4FLACOzdteBj6V15eRxhiuKez7N9I/pJtJd3p17iM/Ix/vztzNcjOvjxGsyunAHNIYzDLSu/fi38slwHtI/wS7urYHSX3+PyK96/4k6bbnauMn1fa/inTNXyC1apaSBqCvr1J8Gqm77m+k7qKXeGMX0khgnqTnSIPsR+YxqXeQBu+XAw8At9F9YpxKShiVx9nAD0hjPU8Ad+Z6dHYpMBF4jNRVeGq+xkXAKODrpAS4CPhXqv9vei9wV76GKaQxq4e7qas1gPylVNabSJoItEfEN5pcj2OAsRHxwWbWw6zZ3BIxW025S+ZkoFq3klmf4iRithrymEoHqWvp102ujlnTuTvLzMxKq1tLRNI2kv4g6f58T/c/5/iWkqZLmp9/9s9xSRonaYHSNBJ7FY41JpefL2lMIb63pDl5n3GdbgU1M7M6q1tLRNIgYFBE3J1v15xFmu7iWGBZRJwr6UzSPe1nSDoE+BLpk6r7kqZo2DffMtgGtJJu6ZsF7B0RT0maQbrL4y7SXSPjIuKm7uo1YMCAGDp0aB2u2MzsrWvWrFlPRMSbpqOp2yyqEbGEPGVDRDwr6QHSJ3VHASNysUnAraRbMUcBl+T74++UtEVORCOA6RGxDEDSdGCkpFuBzSvTN0i6hJSkuk0iQ4cOpa2trecu1MysD5BUdbaIhgysSxoK7ElqMQzMCQbSPeOVGUMH88b72ttzrLt4e5V4tfOPVZrBtK2jo2ONrsXMzF5X9yQiaVPgauC0zhO75VZH3Uf2I2J8RLRGRGtLS7XJQc3MrIy6JpE81cPVwK8iovJJ46W5m6oybvJ4ji/mjfPrDMmx7uJDqsTNzKxB6nl3loBfAA9ExPmFTVNI32FA/nl9IX5MvktrP+CZ3O01DThI6Ytv+gMHAdPytuVKX2gj0qRx1aaDMDOzOqnn15N+gPRdC3Mkzc6xr5O+KOgKSceT5vg5Im+bSrozawFpSu7jACJimaRv8/r3SHyrMshO+tTwRNK8PTexikF1MzPrWX3uw4atra3hu7PMzFaPpFkR0do57mlPzMysNCcRMzMrzUnEzMxKq+fA+luO/sNTc9VLnNW3xubM3ircEjEzs9KcRMzMrDQnETMzK81JxMzMSnMSMTOz0pxEzMysNCcRMzMrzUnEzMxKcxIxM7PSnETMzKw0JxEzMyvNScTMzEpzEjEzs9KcRMzMrLS6JRFJEyQ9LmluIXa5pNn5sbDy3euShkp6sbDtosI+e0uaI2mBpHGSlONbSpouaX7+2b9e12JmZtXVsyUyERhZDETEZyNieEQMB64GrilsfqiyLSJOKsQvBE4AhuVH5ZhnArdExDDglrxuZmYNVLckEhG3A8uqbcutiSOAy7o7hqRBwOYRcWdEBHAJMDpvHgVMysuTCnEzM2uQZo2J7A8sjYj5hdh2ku6RdJuk/XNsMNBeKNOeYwADI2JJXn4MGNjVySSNldQmqa2jo6OHLsHMzJqVRI7ija2QJcC2EbEn8GXg15I2r/VguZXS5ferRsT4iGiNiNaWlpaydTYzs04a/h3rktYFPgXsXYlFxApgRV6eJekhYCdgMTCksPuQHANYKmlQRCzJ3V6PN6L+Zmb2uma0RD4G/DUi/tFNJalFUr+8vD1pAP3h3F21XNJ+eRzlGOD6vNsUYExeHlOIm5lZg9TzFt/LgDuAnSW1Szo+bzqSNw+ofwi4L9/yexVwUkRUBuVPBn4OLAAeAm7K8XOBAyXNJyWmc+t1LWZmVl3durMi4qgu4sdWiV1NuuW3Wvk2YLcq8SeBA9aslmZmtib8iXUzMyvNScTMzEpzEjEzs9KcRMzMrDQnETMzK81JxMzMSnMSMTOz0pxEzMysNCcRMzMrzUnEzMxKcxIxM7PSnETMzKw0JxEzMyvNScTMzEpzEjEzs9KcRMzMrDQnETMzK81JxMzMSqvnd6xPkPS4pLmF2NmSFkuanR+HFLZ9TdICSQ9KOrgQH5ljCySdWYhvJ+muHL9c0vr1uhYzM6uuni2RicDIKvELImJ4fkwFkLQLcCSwa97np5L6SeoH/AT4OLALcFQuC/DdfKwdgaeA4+t4LWZmVkXdkkhE3A4sq7H4KGByRKyIiEeABcA++bEgIh6OiJeBycAoSQI+ClyV958EjO7RCzAzs1VqxpjIKZLuy91d/XNsMLCoUKY9x7qKbwU8HRErO8WrkjRWUpukto6Ojp66DjOzPq/RSeRCYAdgOLAEOK8RJ42I8RHRGhGtLS0tjTilmVmfsG4jTxYRSyvLki4Gbsiri4FtCkWH5BhdxJ8EtpC0bm6NFMubmVmDNLQlImlQYfVwoHLn1hTgSEkbSNoOGAbMAGYCw/KdWOuTBt+nREQAfwA+nfcfA1zfiGswM7PX1a0lIukyYAQwQFI7cBYwQtJwIICFwIkAETFP0hXA/cBK4IsR8Wo+zinANKAfMCEi5uVTnAFMlvQd4B7gF/W6FjMzq65uSSQijqoS7vIffUScA5xTJT4VmFol/jDp7i0zM2sSf2LdzMxKcxIxM7PSnETMzKw0JxEzMyvNScTMzEpzEjEzs9KcRMzMrDQnETMzK81JxMzMSnMSMTOz0pxEzMysNCcRMzMrzUnEzMxKcxIxM7PSnETMzKw0JxEzMyvNScTMzEpzEjEzs9LqlkQkTZD0uKS5hdj3JP1V0n2SrpW0RY4PlfSipNn5cVFhn70lzZG0QNI4ScrxLSVNlzQ//+xfr2sxM7Pq6tkSmQiM7BSbDuwWEbsDfwO+Vtj2UEQMz4+TCvELgROAYflROeaZwC0RMQy4Ja+bmVkD1S2JRMTtwLJOsd9FxMq8eicwpLtjSBoEbB4Rd0ZEAJcAo/PmUcCkvDypEDczswZp5pjIF4CbCuvbSbpH0m2S9s+xwUB7oUx7jgEMjIglefkxYGBXJ5I0VlKbpLaOjo4eqr6ZmTUliUj6N2Al8KscWgJsGxF7Al8Gfi1p81qPl1sp0c328RHRGhGtLS0ta1BzMzMrWrfRJ5R0LPAJ4ID8z5+IWAGsyMuzJD0E7AQs5o1dXkNyDGCppEERsSR3ez3eoEswM7OsppaIpPf0xMkkjQS+ChwWES8U4i2S+uXl7UkD6A/n7qrlkvbLd2UdA1yfd5sCjMnLYwpxMzNrkFq7s34qaYakkyW9rZYdJF0G3AHsLKld0vHAj4HNgOmdbuX9EHCfpNnAVcBJEVEZlD8Z+DmwAHiI18dRzgUOlDQf+FheNzOzBqqpOysi9pc0jDQYPkvSDOB/ImJ6N/scVSX8iy7KXg1c3cW2NmC3KvEngQNqqL6ZmdVJzQPrETEf+AZwBvBhYFz+4OCn6lU5MzNbu9U6JrK7pAuAB4CPAp+MiHfn5QvqWD8zM1uL1Xp31o9I4xJfj4gXK8GIeFTSN+pSMzMzW+vVmkQOBV6MiFcBJK0DbBgRL0TEpXWrnZmZrdVqHRO5GdiosL5xjpmZWR9WaxLZMCKeq6zk5Y3rUyUzM+stak0iz0vaq7IiaW/gxW7Km5lZH1DrmMhpwJWSHgUEvAP4bN1qZWZmvUKtHzacKeldwM459GBEvFK/apmZWW+wOhMwvhcYmvfZSxIRcUldamVmZr1CTUlE0qXADsBs4NUcrnxJlJmZ9VG1tkRagV0qU7ebmZlB7XdnzSUNppuZmf1DrS2RAcD9efbeFZVgRBxWl1qZmVmvUGsSObuelTAzs96p1lt8b5P0TmBYRNwsaWOgX32rZmZma7tap4I/gfSNgz/LocHAdfWqlJmZ9Q61Dqx/EfgAsBz+8QVVb69XpczMrHeoNYmsiIiXKyuS1iV9TqRbkiZIelzS3EJsS0nTJc3PP/vnuCSNk7RA0n2d5uoak8vPlzSmEN9b0py8zzhJqvF6zMysB9SaRG6T9HVgI0kHAlcCv6lhv4nAyE6xM4FbImIYcEteB/g4MCw/xgIXQko6wFnAvsA+wFmVxJPLnFDYr/O5zMysjmpNImcCHcAc4ERgKun71rsVEbcDyzqFRwGT8vIkYHQhfkkkdwJbSBoEHAxMj4hlEfEUMB0YmbdtHhF35g9BXlI4lpmZNUCtd2e9BlycH2tqYEQsycuPAQPz8mBgUaFce451F2+vEn8TSWNJrRu23XbbNay+mZlV1Dp31iNUGQOJiO3X5OQREZLqPpVKRIwHxgO0trZ66hYzsx6yOnNnVWwIfAbYsuQ5l0oaFBFLcpfU4zm+GNimUG5Iji0GRnSK35rjQ6qUNzOzBqlpTCQiniw8FkfED4BDS55zClC5w2oMcH0hfky+S2s/4Jnc7TUNOEhS/zygfhAwLW9bLmm/fFfWMYVjmZlZA9TanbVXYXUdUstklftKuozUihggqZ10l9W5wBWSjgf+DhyRi08FDgEWAC8AxwFExDJJ3wZm5nLfiojKYP3JpDvANgJuyg8zM2uQWruzzissrwQW8vo//y5FxFFdbDqgStkgfaix2nEmABOqxNuA3VZVDzMzq49a7876SL0rYmZmvU+t3Vlf7m57RJzfM9UxM7PeZHXuznovafAb4JPADGB+PSplZma9Q61JZAiwV0Q8CyDpbODGiDi6XhUzM7O1X63TngwEXi6sv8zrnzQ3M7M+qtaWyCXADEnX5vXRvD7/lZmZ9VG13p11jqSbgP1z6LiIuKd+1TIzs96g1u4sgI2B5RHxQ6Bd0nZ1qpOZmfUStX497lnAGcDXcmg94Jf1qpSZmfUOtbZEDgcOA54HiIhHgc3qVSkzM+sdak0iL+dpSQJA0ib1q5KZmfUWtSaRKyT9jPRtgycAN9MzX1BlZma9WC0z8Qq4HHgXsBzYGfhmREyvc93MzGwtt8okkr99cGpEvIf0/eZmZmZA7d1Zd0t6b11rYmZmvU6tn1jfFzha0kLSHVoiNVJ2r1fFzMxs7ddtEpG0bUT8L3Bwg+pjZma9yKpaIteRZu/9u6SrI+KfGlEpMzPrHVY1JqLC8vY9cUJJO0uaXXgsl3SapLMlLS7EDyns8zVJCyQ9KOngQnxkji2QdGZP1M/MzGq3qpZIdLFcWkQ8CAwHkNQPWAxcCxwHXBAR3y+Wl7QLcCSwK7A1cLOknfLmnwAHAu3ATElTIuL+nqinmZmt2qqSyB6SlpNaJBvlZXh9YH3zNTz/AcBDubusqzKjgMkRsQJ4RNICYJ+8bUFEPAwgaXIu6yRiZtYg3XZnRUS/iNg8IjaLiHXzcmV9TRMIpBbGZYX1UyTdJ2mCpP45NhhYVCjTnmNdxd9E0lhJbZLaOjo6eqDaZmYGqzcVfI+StD5pUscrc+hCYAdSV9cS4LyeOldEjI+I1ohobWlp6anDmpn1ebV+TqQePg7cHRFLASo/ASRdDNyQVxcD2xT2G5JjdBM3M7MGaFpLBDiKQleWpEGFbYcDc/PyFOBISRvkL8IaBswAZgLDJG2XWzVH5rJmZtYgTWmJ5KnkDwROLIT/W9Jw0l1gCyvbImKepCtIA+YrgS9GxKv5OKcA04B+wISImNewizAzs+YkkYh4HtiqU+zz3ZQ/BzinSnwqMLXHK2hmZjVpZneWmZn1ck4iZmZWmpOImZmV5iRiZmalOYmYmVlpTiJmZlaak4iZmZXmJGJmZqU5iZiZWWlOImZmVpqTiJmZleYkYmZmpTmJmJlZaU4iZmZWmpOImZmV5iRiZmalOYmYmVlpTiJmZlZa05KIpIWS5kiaLaktx7aUNF3S/Pyzf45L0jhJCyTdJ2mvwnHG5PLzJY1p1vWYmfVFzW6JfCQihkdEa14/E7glIoYBt+R1gI8Dw/JjLHAhpKQDnAXsC+wDnFVJPGZmVn/NTiKdjQIm5eVJwOhC/JJI7gS2kDQIOBiYHhHLIuIpYDowstGVNjPrq5qZRAL4naRZksbm2MCIWJKXHwMG5uXBwKLCvu051lX8DSSNldQmqa2jo6Mnr8HMrE9bt4nn/mBELJb0dmC6pL8WN0ZESIqeOFFEjAfGA7S2tvbIMc3MrIktkYhYnH8+DlxLGtNYmrupyD8fz8UXA9sUdh+SY13FzcysAZqSRCRtImmzyjJwEDAXmAJU7rAaA1yfl6cAx+S7tPYDnsndXtOAgyT1zwPqB+WYmZk1QLO6swYC10qq1OHXEfFbSTOBKyQdD/wdOCKXnwocAiwAXgCOA4iIZZK+DczM5b4VEcsadxlmZn1bU5JIRDwM7FEl/iRwQJV4AF/s4lgTgAk9XUczM1u1te0WXzMz60WcRMzMrDQnETMzK62ZnxMxq79084bVQ/gjV+aWiJmZrQEnETMzK81JxMzMSnMSMTOz0pxEzMysNCcRMzMrzUnEzMxKcxIxM7PSnETMzKw0JxEzMyvNScTMzEpzEjEzs9KcRMzMrDQnETMzK63hSUTSNpL+IOl+SfMk/XOOny1psaTZ+XFIYZ+vSVog6UFJBxfiI3NsgaQzG30tZmZ9XTO+T2Ql8JWIuFvSZsAsSdPztgsi4vvFwpJ2AY4EdgW2Bm6WtFPe/BPgQKAdmClpSkTc35CrMDOzxieRiFgCLMnLz0p6ABjczS6jgMkRsQJ4RNICYJ+8bUFEPAwgaXIu6yRiZtYgTR0TkTQU2BO4K4dOkXSfpAmS+ufYYGBRYbf2HOsqXu08YyW1SWrr6OjowSswM+vbmpZEJG0KXA2cFhHLgQuBHYDhpJbKeT11rogYHxGtEdHa0tLSU4c1M+vzmvId65LWIyWQX0XENQARsbSw/WLghry6GNimsPuQHKObuJmZNUAz7s4S8AvggYg4vxAfVCh2ODA3L08BjpS0gaTtgGHADGAmMEzSdpLWJw2+T2nENZiZWdKMlsgHgM8DcyTNzrGvA0dJGg4EsBA4ESAi5km6gjRgvhL4YkS8CiDpFGAa0A+YEBHzGnkhZmZ9XTPuzvoToCqbpnazzznAOVXiU7vbz8zM6sufWDczs9KcRMzMrDQnETMzK81JxMzMSnMSMTOz0pryYUMzs2qGnnljs6vwlrXw3EPrcly3RMzMrDQnETMzK81JxMzMSnMSMTOz0pxEzMysNCcRMzMrzUnEzMxKcxIxM7PSnETMzKw0JxEzMyvNScTMzEpzEjEzs9J6fRKRNFLSg5IWSDqz2fUxM+tLenUSkdQP+AnwcWAX4ChJuzS3VmZmfUevTiLAPsCCiHg4Il4GJgOjmlwnM7M+o7d/n8hgYFFhvR3Yt3MhSWOBsXn1OUkPNqBua4MBwBPNrkQtdLaaXYW1Qa95vQCQXzN60Wum767xId5ZLdjbk0hNImI8ML7Z9Wg0SW0R0drselht/Hr1Pn7Nen931mJgm8L6kBwzM7MG6O1JZCYwTNJ2ktYHjgSmNLlOZmZ9Rq/uzoqIlZJOAaYB/YAJETGvydVam/S5Lrxezq9X79PnXzNFRLPrYGZmvVRv784yM7MmchIxM7PSnETWUpJOlfSApF9J2kDSzZJmS/psD57jLz11LFs9kk6SdExePlbS1oVtCyUNaF7tehdJQyXNXc19RveG2S0kjZB0Q7Pr0Z1ePbD+Fncy8LGIaJe0H0BEDO/JE0TE+3vyeFa7iLiosHosMBd4dE2PK2ndiFi5psfpA0YDNwD3N7sivZ1bImsBSV+WNDc/TpN0EbA9cJOkM4BfAu/NLZEdJO0t6TZJsyRNkzQoH+dWSd+VNEPS3yTtn+O75thsSfdJGpbjz+WfkyUdWqjPREmfltRP0vckzcz7ndjo56a3yO+GH5B0saR5kn4naaP8ev02v1Z/lPSuXP5sSadL+jTQCvwqvz4b5UN+SdLdkuYU9tlE0oT8Wt4jaVSOHytpiqTfA7c04/rXAv06P/cAkk7Iv7/3Srpa0saS3g8cBnyv8DdV9XUqkvThXH52fv43yy2F2yXdmCeCvUjSOrn8QZLuyK/jlZI2zfGu/n53zD0O9+Z9dsin3lTSVZL+mnsm1q6pAiLCjyY+gL2BOcAmwKbAPGBPYCEwIJcZAdyQl9cD/gK05PXPkm5tBrgVOC8vHwLcnJd/BHwuL68PbJSXn8s/DwcmFbYvAjYiTRXzjRzfAGgDtmv2c7Y2PoChwEpgeF6/Ajia9E99WI7tC/w+L58NnF543VoLx1oIfCkvnwz8PC//J3B0Xt4C+Fv+vTmWNOXPls1+Htam5z4vb1Uo953C8zoR+HRhW9XXqdN5fgN8IC9vSurJGQG8RHrT1w+YDnyaNB3K7cAmufwZwDdX8fd7F3B4Xt4Q2Dgf/xnSB6nXAe4APtjs57z4cHdW830QuDYingeQdA2wfzfldwZ2A6bnNyT9gCWF7dfkn7NIf1yQfvH+TdIQ4JqImN/pmDcBP5S0ATASuD0iXpR0ELB7frcM8DZgGPDIal9l3/BIRMzOy5Xn//3AlYU3jxvUeKzi6/ipvHwQcJik0/P6hsC2eXl6RCwrWe+3gmrPPcBukr5DSrqbkj5T9ga5hVDL6/Rn4HxJvyL9HbXn8jMi4uF8rMtIf9MvkWYW/3Musz7p77Dq36+kzYDBEXEtQES8lI9XOX57Xp+dr+1Pq/Xs1JGTSO8jYF5EvK+L7Svyz1fJr29E/FrSXcChwFRJJ0bE7ys7RMRLkm4FDia9M5pcONeXIuJNf3hW1YrC8qvAQODpKDeW9abXkfR6/FNEvGECUUn7As+XOMdbSefnvtItOBEYHRH3SjqW9M6+s3Wo4XWKiHMl3Uhq5f9Z0sGVTZ2Lkl6r6RFxVHGDpPdQ5e83J5GudL62ter/tsdEmu+PwOjcV7sJqWvpj92UfxBokfQ+AEnrSdq1uxNI2h54OCLGAdcDu1cpdjlwHKkV9Nscmwb8P0nr5ePslOtotVkOPCLpMzQqn0oAAAOKSURBVABK9qhS7lmgu38iFdNIYyXKx9uzx2r61rUZ6Z3+esDnCvF/POcRUdPrJGmHiJgTEd8lTblUGTfZR2nqpXVIb8L+BNwJfEDSjnnfTSTtRBd/vxHxLNAuaXSObyBp4x5+LurCSaTJIuJu0rulGaQ+0Z9HxD3dlH+Z1Of6XUn3ArNJTfHuHAHMzU3h3YBLqpT5HfBh0jjKyzn2c9LdK3cr3UL5M9ayd0G9wOeA4/NrNY/q33czEbhIbxxYr+bbpD71+yTNy+vWvX8n/V39GfhrIT4Z+Nc8QL4Dtb1Opynd/HIf8AqpGxhSQvkx8ACpq/faiOggjVVdlsvfAbxrFX+/nwdOzeX/AryjJ56AevO0J2ZmJUkaQbpB4hPNrkuzuCViZmaluSViZmaluSViZmalOYmYmVlpTiJmZlaak4hZNyS9Q2lusYfyXEdT8+dlVnvm2NU459mFT6XXUv65eh7frDu+59+sC/lDfdeS5hU7Msf2IH0SfVEz62a2tnBLxKxrHwFeicK07RFxb0S8YUaB3Cr5Y5559W6lWWKRNEhphtfZ+UNq+yvNjDwxr8+R9C+1VkbSdbk1NE/S2E7bLsjxWyS15NgqZ6Y1W1NOImZd2400md+qPA4cGBF7kaa9GJfj/weYludk2oP06eThpIn2douI9wD/sxr1+UJE7E2aOv5USVvl+CZAW0TsCtwGnJXj40lzn+0NnA78dDXOZVYTd2eZrbn1gB9LGk6aIG+nHJ8JTMjzNl0XEbMlPQxsL+lHwI2k6WZqdaqkw/PyNqQZlZ8EXiPNfQbpu2euWY2Zac3WiFsiZl2bR/q+l1X5F2ApqbXRSpr2m4i4HfgQsBiYKOmYiHgql7sVOIk0P9kq5ek1Pga8LyL2AO4hTQVfTVCYmbbweHct5zJbHU4iZl37PbBBcfxB0u7K3xhZ8DZgSUS8RppEr18u+05gaURcTEoWeyl9d/o6EXE18A1grxrr8jbgqYh4IY9t7FfYtg5pUj9IXWh/qnVmWrM15SRi1oVIcwIdDnws3+I7D/gv4LFORX8KjMmzsr6L17/bYwRwr6R7SGMlPwQGA7fmGZV/CXyti9N/Q1J75UGann9dSQ8A55KmGq94njQd+Vzgo8C3cryWmWnN1ojnzjIzs9LcEjEzs9KcRMzMrDQnETMzK81JxMzMSnMSMTOz0pxEzMysNCcRMzMr7f8DPPam7iXk7nQAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "# Plot distribution of samples across classes\n",
        "labels = df['class']\n",
        "bins = ['offensive', 'neither', 'hate speech']\n",
        "\n",
        "y_pos = np.arange(len(bins))\n",
        "value_counts = labels.value_counts()\n",
        "\n",
        "plt.bar(y_pos, value_counts, align='center', color=['g', 'r', '#1f77b4'])\n",
        "\n",
        "plt.xticks(y_pos, bins)\n",
        "plt.xlabel(\"Class Label\")\n",
        "plt.ylabel(\"Frequency\")\n",
        "plt.title(\"Frequency of Class Labels\")\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g2csNiUqLLON"
      },
      "source": [
        "The bar chart shows the imbalanced nature of the classification problem; most tweets containing \"hateful\" words, as defined by Hatebase, were coded as offensive by CF coders. More tweets were considered neither hate speech nor offensive than were considered hate speech."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zzBYrPoSLLOO"
      },
      "source": [
        "### Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P9ahx9HDLLON"
      },
      "outputs": [],
      "source": [
        "# Select data for preprocessing and transformation\n",
        "tweets=df.tweet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fP6SQhOafUnp",
        "outputId": "0510f5ff-4dd1-4a66-a69b-665ab8133212"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        }
      ],
      "source": [
        "# Create list of stopwords \n",
        "nltk.download('stopwords')\n",
        "stopwords = nltk.corpus.stopwords.words(\"english\")\n",
        "other_exclusions = [\"#ff\", \"ff\", \"rt\"]\n",
        "stopwords.extend(other_exclusions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oua7Ybzl0U66"
      },
      "outputs": [],
      "source": [
        "# Create reusable stemmer\n",
        "stemmer = nltk.stem.porter.PorterStemmer()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mL-YdNXuLLOP"
      },
      "outputs": [],
      "source": [
        "# Define functions for preprocessing data\n",
        "def preprocess(text_string):\n",
        "    \"\"\"\n",
        "    Accepts a text string and replaces:\n",
        "    1) urls with URLHERE\n",
        "    2) lots of whitespace with one instance\n",
        "    3) mentions with MENTIONHERE\n",
        "\n",
        "    This allows us to get standardized counts of urls and mentions\n",
        "    Without caring about specific people mentioned\n",
        "    \"\"\"\n",
        "    space_pattern = '\\s+'\n",
        "    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'\n",
        "        '[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\n",
        "    mention_regex = '@[\\w\\-]+'\n",
        "    parsed_text = re.sub(space_pattern, ' ', text_string)\n",
        "    parsed_text = re.sub(giant_url_regex, '', parsed_text)\n",
        "    parsed_text = re.sub(mention_regex, '', parsed_text)\n",
        "    return parsed_text\n",
        "\n",
        "def tokenize(tweet):\n",
        "    \"\"\"Removes punctuation & excess whitespace, sets to lowercase,\n",
        "    and stems tweets. Returns a list of stemmed tokens.\"\"\"\n",
        "    tweet = \" \".join(re.split(\"[^a-zA-Z]*\", tweet.lower())).strip()\n",
        "    tokens = [stemmer.stem(t) for t in tweet.split()]\n",
        "    return tokens\n",
        "\n",
        "def basic_tokenize(tweet):\n",
        "    \"\"\"Same as tokenize but without the stemming\"\"\"\n",
        "    tweet = \" \".join(re.split(\"[^a-zA-Z.,!?]*\", tweet.lower())).strip()\n",
        "    return tweet.split()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1gLmeeK86S8b"
      },
      "source": [
        "### Feature Engineering"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oKTPCht46_Y4"
      },
      "source": [
        "####Automatic Feature Extraction\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3ciSa7cILLOQ",
        "outputId": "3b3d3c37-079d-4081-db1a-dbe587c05b12"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/feature_extraction/text.py:401: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['b', 'c', 'e', 'f', 'g', 'h', 'j', 'l', 'n', 'p', 'r', 'u', 'v', 'w'] not in stop_words.\n",
            "  % sorted(inconsistent)\n"
          ]
        }
      ],
      "source": [
        "# Construct tfidf matrix and get relevant scores\n",
        "vectorizer = TfidfVectorizer(\n",
        "    tokenizer=tokenize,\n",
        "    preprocessor=preprocess,\n",
        "    ngram_range=(1, 3),\n",
        "    stop_words=stopwords,\n",
        "    use_idf=True,\n",
        "    smooth_idf=False,\n",
        "    norm=None,\n",
        "    decode_error='replace',\n",
        "    max_features=10000,\n",
        "    min_df=5,\n",
        "    max_df=0.75\n",
        "    )\n",
        "\n",
        "tfidf = vectorizer.fit_transform(tweets).toarray()\n",
        "vocab = {v:i for i, v in enumerate(vectorizer.get_feature_names_out())}\n",
        "idf_vals = vectorizer.idf_\n",
        "idf_dict = {i:idf_vals[i] for i in vocab.values()} #keys are indices; values are IDF scores"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yB8bQvC8LLOQ",
        "outputId": "e42ac1aa-b6d0-4c5f-c7d1-8b25aa4a7d4d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ]
        }
      ],
      "source": [
        "# Get POS tags for tweets and save as a string\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "tweet_tags = []\n",
        "for t in tweets:\n",
        "    tokens = basic_tokenize(preprocess(t))\n",
        "    tags = nltk.pos_tag(tokens)\n",
        "    tag_list = [x[1] for x in tags]\n",
        "    tag_str = \" \".join(tag_list)\n",
        "    tweet_tags.append(tag_str)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "nIgc3oc_LLOR"
      },
      "outputs": [],
      "source": [
        "# Construct POS TF matrix and get vocab dict\n",
        "pos_vectorizer = TfidfVectorizer(\n",
        "    tokenizer=None,\n",
        "    lowercase=False,\n",
        "    preprocessor=None,\n",
        "    ngram_range=(1, 3),\n",
        "    stop_words=None,\n",
        "    use_idf=False,\n",
        "    smooth_idf=False,\n",
        "    norm=None,\n",
        "    decode_error='replace',\n",
        "    max_features=5000,\n",
        "    min_df=5,\n",
        "    max_df=0.75,\n",
        "    )\n",
        "\n",
        "pos = pos_vectorizer.fit_transform(pd.Series(tweet_tags)).toarray()\n",
        "pos_vocab = {v:i for i, v in enumerate(pos_vectorizer.get_feature_names_out())}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7HBwr5At6mud"
      },
      "source": [
        "#### Manual Feature Construction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A9wjv7Di0lYE"
      },
      "outputs": [],
      "source": [
        "# Create reusable sentiment analyzer\n",
        "sentiment_analyzer = VS()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "s-x-KmYQLLOS"
      },
      "outputs": [],
      "source": [
        "# Define functions for devising features\n",
        "def count_twitter_objs(text_string):\n",
        "    \"\"\"\n",
        "    Accepts a text string and replaces:\n",
        "    1) urls with URLHERE\n",
        "    2) lots of whitespace with one instance\n",
        "    3) mentions with MENTIONHERE\n",
        "    4) hashtags with HASHTAGHERE\n",
        "\n",
        "    This allows us to get standardized counts of urls and mentions\n",
        "    Without caring about specific people mentioned.\n",
        "    \n",
        "    Returns counts of urls, mentions, and hashtags.\n",
        "    \"\"\"\n",
        "    space_pattern = '\\s+'\n",
        "    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'\n",
        "        '[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\n",
        "    mention_regex = '@[\\w\\-]+'\n",
        "    hashtag_regex = '#[\\w\\-]+'\n",
        "    parsed_text = re.sub(space_pattern, ' ', text_string)\n",
        "    parsed_text = re.sub(giant_url_regex, 'URLHERE', parsed_text)\n",
        "    parsed_text = re.sub(mention_regex, 'MENTIONHERE', parsed_text)\n",
        "    parsed_text = re.sub(hashtag_regex, 'HASHTAGHERE', parsed_text)\n",
        "    return(parsed_text.count('URLHERE'),parsed_text.count('MENTIONHERE'),parsed_text.count('HASHTAGHERE'))\n",
        "\n",
        "def other_features(tweet):\n",
        "    \"\"\"This function takes a string and returns a list of features.\n",
        "    These include Sentiment scores, Text and Readability scores,\n",
        "    as well as Twitter specific features\"\"\"\n",
        "    sentiment = sentiment_analyzer.polarity_scores(tweet)\n",
        "    \n",
        "    words = preprocess(tweet) # Get text only\n",
        "    \n",
        "    syllables = textstat.syllable_count(words)\n",
        "    num_chars = sum(len(w) for w in words)\n",
        "    num_chars_total = len(tweet)\n",
        "    num_terms = len(tweet.split())\n",
        "    num_words = len(words.split())\n",
        "    avg_syl = round(float((syllables+0.001))/float(num_words+0.001),4)\n",
        "    num_unique_terms = len(set(words.split()))\n",
        "    \n",
        "    # Modified FK grade, where avg words per sentence is just num words/1\n",
        "    FKRA = round(float(0.39 * float(num_words)/1.0) + float(11.8 * avg_syl) - 15.59,1)\n",
        "    # Modified FRE score, where sentence fixed to 1\n",
        "    FRE = round(206.835 - 1.015*(float(num_words)/1.0) - (84.6*float(avg_syl)),2)\n",
        "    \n",
        "    twitter_objs = count_twitter_objs(tweet)\n",
        "    retweet = 0\n",
        "    if \"rt\" in words:\n",
        "        retweet = 1\n",
        "    features = [FKRA, FRE,syllables, avg_syl, num_chars, num_chars_total, num_terms, num_words,\n",
        "                num_unique_terms, sentiment['neg'], sentiment['pos'], sentiment['neu'], sentiment['compound'],\n",
        "                twitter_objs[2], twitter_objs[1],\n",
        "                twitter_objs[0], retweet]\n",
        "    return features\n",
        "\n",
        "def get_feature_array(tweets):\n",
        "    feats=[]\n",
        "    for t in tweets:\n",
        "        feats.append(other_features(t))\n",
        "    return np.array(feats)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ov78PfltLLOT"
      },
      "outputs": [],
      "source": [
        "# Construct features\n",
        "other_feats = get_feature_array(tweets)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BTOyivbhxW0o"
      },
      "source": [
        "### Feature Selection"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "sVht_1gaLLOT"
      },
      "outputs": [],
      "source": [
        "# Join automatically extracted features and manually constructed features together\n",
        "X = np.concatenate([tfidf,pos,other_feats],axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wKJPahrqLLOT",
        "outputId": "5aa284c8-f66c-4593-ddbf-131be8a06c6c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(24783, 4023)"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ],
      "source": [
        "# Get dimensions of the data\n",
        "X.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8xcsLQ9Z-GeY"
      },
      "source": [
        "The data is high dimensional. There is an exponential decrease in predictive power for each additional variable in a multivariate model, known as the curse of dimensionality. Therefore, the dimensionality of the data should be reduced."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qmRY5E7NLLOU"
      },
      "source": [
        "### Model Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "UFOwkOMQLLOV"
      },
      "outputs": [],
      "source": [
        "# Encode class labels\n",
        "y = df['class'].astype(int)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7MtWbgBv-Hy4"
      },
      "outputs": [],
      "source": [
        "# Reduce dimensionality of data. Specify solver as 'liblinear' because default solver changed to 'lbfgs' in 0.22.\n",
        "feature_transformer = SelectFromModel(LogisticRegression(class_weight='balanced',penalty=\"l1\", C=0.01, solver='liblinear'))\n",
        "\n",
        "X_ = feature_transformer.fit_transform(X,y)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TAX5BIHbI0E5",
        "outputId": "fe1017cd-84c1-4fb1-a18e-c54d565e399a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(24783, 465)"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ],
      "source": [
        "# Get reduced dimensions of the data\n",
        "X_.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WO8bcXNPA60Q"
      },
      "outputs": [],
      "source": [
        "# Generate train/test split of 9/1. Initialise with random seed for reproducability.\n",
        "X_train, X_test, y_train, y_test = train_test_split(X_, y, random_state=42, test_size=0.1, stratify=y)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ywaN85tJSsOR"
      },
      "source": [
        "As classification problem is imbalanced, it's desirable for the train/test split to preserve the same proportions of examples in each class as observed in the original dataset [(Brownlee, 2020)](https://machinelearningmastery.com/train-test-split-for-evaluating-machine-learning-algorithms//)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pvrfVFjoWAlt"
      },
      "outputs": [],
      "source": [
        "# Create model. Change hyperparameters to ensure control variables remain constant. Change solver to the new default solver, lbfgs.\n",
        "# Increase `max_iter` as lbfgs takes longer to converge. Change framework used for multi class classification to multinomial.\n",
        "# See comments in `start_flower_client` for details.\n",
        "classifier = LogisticRegression(class_weight='balanced',penalty='l2', C=0.01, max_iter=20000, multi_class=\"multinomial\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "popozOSPgSTm"
      },
      "outputs": [],
      "source": [
        "# Train model\n",
        "model = classifier.fit(X_train, y_train)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xIaJSdpALLOW"
      },
      "source": [
        "### Model Testing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QCpkWAWRLLOW"
      },
      "outputs": [],
      "source": [
        "# Test model\n",
        "y_preds = model.predict(X_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JVR1H_vMLLOW",
        "outputId": "c81d0f69-0818-4e0f-fae9-cb0945430845"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.24      0.69      0.36       143\n",
            "           1       0.96      0.74      0.84      1920\n",
            "           2       0.61      0.86      0.72       416\n",
            "\n",
            "    accuracy                           0.76      2479\n",
            "   macro avg       0.61      0.77      0.64      2479\n"
          ]
        }
      ],
      "source": [
        "# Generate test report\n",
        "report = classification_report(y_test, y_preds)\n",
        "print(report.rsplit(\"\\n\",2)[0])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4C3oYzjPMiDA"
      },
      "source": [
        "There is a class-imbalance in the dataset; the hate speech class (0) and neither class (2) are most important to get right, and they are under-represented. Offensive language class (1) is over-represented. Therefore, macro average is preferable over weighted average, which is misleading [(Khalusova, 2021)](https://www.mariakhalusova.com/posts/2019-04-17-ml-model-evaluation-metrics-p2/#micro--macro--weighted-averaged-precision).\n",
        "\n",
        "F1-score is used rather than AUROC because AUROC averages over all possible discrimination thresholds, which is misleading in the case of class-imbalanced data [(2018, Almardeny)](https://stackoverflow.com/questions/44172162/f1-score-vs-roc-auc). In contrast, F-1 score maintains a balance between Precision and Recall."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2iA7YFi7LLOX"
      },
      "outputs": [],
      "source": [
        "# Define function for plotting confusion matrix\n",
        "def plot_confusion_matrix(y, y_preds):\n",
        "\n",
        "  matrix = confusion_matrix(y,y_preds)\n",
        "  matrix_proportions = np.zeros((3,3))\n",
        "  for i in range(0,3):\n",
        "    matrix_proportions[i,:] = matrix[i,:]/float(matrix[i,:].sum())\n",
        "  names=['Hate','Offensive','Neither']\n",
        "  confusion_df = pd.DataFrame(matrix_proportions, index=names,columns=names)\n",
        "  plt.figure(figsize=(5,5))\n",
        "  seaborn.heatmap(confusion_df,annot=True,annot_kws={\"size\": 12},cmap='gist_gray_r',cbar=False, square=True,fmt='.2f')\n",
        "  plt.ylabel(r'True categories',fontsize=14)\n",
        "  plt.xlabel(r'Predicted categories',fontsize=14)\n",
        "  plt.tick_params(labelsize=12)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 341
        },
        "id": "hy47YnA-EY5M",
        "outputId": "487f057f-2532-4136-9dad-bd4c0aefa2e1"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 360x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUQAAAFECAYAAABf6kfGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5wURfr48c/DLrCBYEA9QBAkCnhkEQEVVEBOAT09wTs98Sv+0BMFT9QznWJExJwwHKiAcJ6YTwFFFEURJIgimUWCgHqkZZeNz++Pqh1m8yzs9OzC83695sVMd3V1dbPzTFVXd5WoKsYYY6BKrAtgjDEVhQVEY4zxLCAaY4xnAdEYYzwLiMYY41lANMYYLz7WBTgQXbp0sXuFAvDaa6/FugiHjaOOOirWRThs1KlTR4pbZzVEY4zxLCAaY4xnAdEYYzwLiMYY41lANMYYzwKiMcZ4FhCNMcazgGiMMZ4FRGOM8SwgGmOMZwHRGGM8C4jGGONZQDTGGM8CojHGeBYQjTHGs4BojDGeBURjjPEsIBpjjGcB0RhjPAuIxhjjWUA0xhjPAqIxxngWEI0xxrOAaIwxngVEY4zxLCAaY4xnAdEYYzwLiMYY41lANMYYzwKiMcZ4FhCNMcazgGiMMZ4FRGOM8SwgGmOMF3hAFJHqInK/iKwTkV1+WW8RuS7oshhjTLhY1BAfA9oAfwbUL/sBuCYGZTHGmJD4GOzzAqCpqu4VkVwAVd0sIvVjUBZjjAmJRQ0xkwKBWESOAX6LQVmMMSYkFgHxDeAVEWkMICJ1gaeBqTEoizHGhMSiyXwbMAZYBiQBq4EXgdExKMtBqVWrFrfffjtdunRh586dPPvss8ycObPItC1atGDkyJG0aNGCffv2MXHiRKZNmwbAySefzMiRI2nUqBFbtmxh7NixLF26NMhDqfD27NnDk08+yeLFi6lVqxaXX345Z555ZqF03333HVOnTmXt2rXUqFGDl19+Od/6devWMX78eFJSUkhMTKRv374MGjQooKOo+Hbv3s2DDz7IN998Q+3atRk2bBi9e/culO7bb79lwoQJrFq1ipo1a/Lmm2/mW3/dddexfv16MjMzqVevHldddRU9evQI6jAOWOABUVUzgZHASN9U/lVVtZTNKqRRo0aRlZXFueeeS/PmzXn00UdZvXo169evz5eudu3aPP744zz++OPMnj2bqlWrcuyxxwIuqI4bN46HHnqIOXPm0Lt3bx555BEuvPBC9uzZE4vDqpCef/554uPjee2111i3bh2jR4+mcePGnHDCCfnSJSQkcPbZZ3P66afzxhtvFMrnkUce4dRTT+WBBx5g+/bt3HLLLTRu3JguXboEdSgV2rhx44iPj+e9995j9erVjBo1iqZNm3LiiSfmS5eYmMh5551HRkYGr776aqF8RowYQaNGjYiPj+eHH37ghhtuYOrUqdSpUyeoQzkgsbjt5n9571X1l7xgKCLbgy7LwUhISKBnz56MHz+e9PR0li5dyty5czn33HMLpb300kuZP38+M2bMICsri7S0NFJSUgBXO/ztt9+YPXs2ubm5fPTRR+zcubPI2s/hat++fcybN4+//OUvJCYm0rp1a0455RQ+/fTTQmmbN29Or169+N3vfldkXtu2bePMM88kLi6OunXr0qpVK3766adoH0KlkJ6ezpw5cxg6dChJSUm0bduW7t27M2PGjEJpW7VqRd++falXr16ReTVt2pT4eFffEhFycnLYvr3if8VjcQ2xasEFIlIViItBWQ5Yw4YNycnJYePGjaFlq1evLvRLCtCmTRt2797Niy++yIcffsgjjzzCcccdF1ovIvnSiwhNmjSJXuErmc2bN1OlShXq199/I0Ljxo0PKJANGDCA2bNnk52dzaZNm1ixYgVt27Ytz+JWWhs3biQuLo6GDRuGljVt2rRQiydSo0aNomfPngwdOpT27dvTsmXL8ipq1ATWZBaRubj7DhNE5PMCq48H5gVVlvKQlJTE3r178y1LTU0lKSmpUNpjjz2WFi1aMHz4cNauXct1113Hvffey9VXX82yZcuoU6cOvXv35pNPPqFPnz7Ur1+fhISEoA6lwtu3b1+h85qcnEx6enqZ8+rcuTOPPfYYb731Frm5uQwaNIjmzZuXV1ErtbS0NJKTk/Mtq1GjBmlpaQeU39ixY8nOzmbBggVs2LCBKlUq/oNxQZbwJeBfQDbwctjrJdxN2ReWtLGIXC0iC0VkYUWoehf1x5OcnFzkH09GRgafffYZP/74I5mZmbz00ku0bduW5ORkdu/ezahRoxg8eDAffvghXbt2ZcGCBZWieRGUhISEQuc1LS2NxMTEMuWzZ88e/vnPfzJo0CCmT5/OhAkTWLx4MR988EF5FrfSKupHfu/evUX+yEcqPj6erl278s033zB37tyDLWLUBVZDVNVXAETka1VdcQDbvwC8ANClS5eYd8L89NNPxMXF0aBBg1CzuVmzZqxbt65Q2jVr1lBSv9HixYsZMmQIAHFxcUyfPp3JkydHp+CVUP369cnNzWXLli2ha1br16/P17SLxNatW6lSpQq9evUCoE6dOvTo0YOFCxfyhz/8odzLXdk0aNAgdBmoQYMGgPvbbdy48UHnnZOTw+bNmw86n2gLvA6rqitE5DgROV9EhojIlXmvoMtyMPbt28ecOXO4+uqrSUhI4Pe//z2nn346H374YaG07733HmeccQbNmjUjLi6OK6+8kiVLloR+jZs3b05cXBzJyclcf/31bNu2jfnz5wd9SBVWQkICXbt2ZfLkyezbt4/ly5czf/58evbsWShtbm4umZmZZGdno6pkZmaSlZUFELoGOWfOHHJzc9mxYwdz584tly/8oSAxMZEzzjiDl156ifT0dL777jvmzp1Lnz59CqXNzc0lIyMjdJ4zMjJC53nDhg189dVXofUzZsxgyZIltG/fPuhDKjMJ+o4XERkITMLdf9ga9xxzG+ALVS38F16EilBDBHfLzB133MEpp5zCrl27eOaZZ5g5cybt2rXjsccey/eFvfDCCxkyZAgJCQksXbqUhx9+ONQsvvfeeznttNMA+Oqrrxg3bhw7duyIyTGFe+2112JdhJA9e/bwxBNPsGTJEmrWrMlf//pXzjzzTH744Qfuvvvu0C02y5Yt47bbbsu3bZs2bXjwwQcBWLp0KRMnTmTLli1Uq1aNU045haFDh8b8mu1RRx0V0/3n2b17Nw888AALFizIdx/ikiVLuOmmm/j4448BWLRoEcOHD8+3bfv27Xn66adJSUnh/vvvZ/369cTFxXH88cdz+eWXc8YZZ8TikAqpU6eOFLcuFgHxe+AeVX1DRHao6pEiMgRorao3RZJHRQmIh7qKFBAPdRUlIB4OSgqIsej2aaiqBe+YfQW4PAZlMcaYkFgExO0ikncTXoqIdAWaUMnuQzTGHHpiERBfBLr7948BnwJLgediUBZjjAmJxbPMY8Levyoic4BkVf0x6LIYY0y4WDypUtx6VPX0oMpjjDEFBVlDfCnsvQDPANcGuH9jjClR4E+q5BGRRwsuM8aYWKr4T1sbY0xALCAaY4wXZKdKr4L7FpGeuOuJAKjq7KDKY4wxBQXZqfJygc+/4YYDy6NA4dFVjTEmIEF2qtiQIsaYCs2uIRpjjGcB0RhjPAuIxhjjWUA0xhjPAqIxxngWEI0xxrOAaIwxngVEY4zxLCAaY4xnAdEYYzwLiMYY41lANMYYzwKiMcZ4FhCNMcazgGiMMZ4FRGOM8SwgGmOMZwHRGGM8C4jGGONZQDTGGM8CojHGeAcUEEUkUUTOFpETyrtAxhgTKxEFRBGZKCLX+vfVgG+AmcBKETk3iuUzxpjARFpD7AN87d/3B2oCvwPu9i9jjKn0Ig2IRwLb/fu+wJuquh2YCrSKRsGMMSZokQbErUAbEYnD1RY/9strAFnRKJgxxgQtPsJ0/wKmAVuAHOATv7wLsCIK5TLGmMBFFBBVdbSI/AA0BN5Q1Uy/KhsYE63CGWNMkERVY12GMtu8eXPlK3Ql1KtXr1gX4bCxYMGCWBfhsFGrVi0pbl3E9yGKyLki8r6ILBeRBn7ZVSJyVnkU0hhjYi3S+xD/DPwbWA00Bqr6VXHAzdEpmjHGBCvSGuLNwFBVHYm7bpjna6BduZfKGGNiINKA2Az4qojlqUCt8iuOMcbETqQBcQvQvIjlpwNry684xhgTO5EGxBeAJ0Wkm//cQET+CjwMPBeVkhljTMAivQ/xYRGpDcwCEoBPgQzgEVV9JorlM8aYwET6pAqqeruI3I97drkKsFxVU6NWMmOMCVjEARFAVdOAhVEqizHGxFSxAVFE3gX+oqq7/ftiqWr/ci+ZMcYErKQa4m9A3iNy/wt7b4wxh6RiA6KqDgn7eC2Qoao50S+SMcbERqm33fgxEHcBLaJfHGOMiZ1SA6KvFW4AqkW/OMYYEzuR3ph9L/CQiNSJZmGMMSaWIr3t5ibcKDebRWQTsDd8par+vrwLZowxQYs0IP4nqqUwxpgKINJH9+6JdkGMMSbWyvSkioj0wj26p8APqjonGoUyxphYiCggikh94C2gI24oMIB6IrIQuEBVtxS7cdH5HQ30A+r6gSPqAVVUdVNZ8jHGmPIUaS/zk7jpR5uqagNVbYAbNDbHr4uYiJwBrAT+DNzpFzfDhhEzxsRYpE3mc4AzVXV93gJVXSci17N/juZIPQ5coqqfiMgOv2w+cEoZ8zHGmHIV8ax7FP0s84E839xIVfOCaN72mZTxeqYxxpS3SAPiJ8BTedOPAohIQ1xtr6w1xOUi0qfAsrOBZWXMxxhjylWktbLrgXeBdSIS6lTBBbHBZdzn34H3ReQDIFFExgPnAwPKmI8xxpSrSO9D3CgiHXA1uZZ+8Y+q+nFZd6iqX4tIW1ynyr+AjcAp1sNsjIm1skwhoLg5VWYdzA5FpJ2qLsFNUGWMMRVGpPch3lXMKgX2AWuAj1Q1PYLsZorIL8DrwOTwnmtjjImlSGuIFwMNgWTCbszGDfLwC9AA2C4iZ6jqulLyqgv0xV17XCoiPwBTgGmqur2M5TfGmHITaS/zOGAB7paZhqraEGiEu39wNC44rgIeLS0jVc1R1Q9U9S/AccATwEW4a4nGGBMzkQbEfwI3hnd8+Pc3A6NV9TfgdqBrpDsWkQTgPOASoBMwN9JtjTEmGiINiMfhJqgvqDpwrH+/DUgqLSMR6Scik4DtuFtwPgOaqOrZEZbFGGOiItJriB8D40XkauBbv6wj7vnjvF7nk4FIOkgewXWotFfVtWUoqzHGRFWkAfEq4FXcNcO8mfeqADOBof7zHtzI2iVS1VZlLKMxxgQi0huztwN9RaQF+2ffW6Gqq8LSfFrc9iJyu6re79+PLmE/xd3eY4wxUVemARVUdaWI7AR+UdXcMmx6fNj7BsWmMsaYGIr0xuyqwP3ANUAi0Bz3XPMYYIOqPlvS9qp6Tdj7IQdeXGOMiZ6y3HZzPvAXICNs+TfAFWXZoYi0EpHj/PsaInKPiPxTRErtoTbGmGiKNCAOBoap6jtAeFP5e1xtsSxeB47w7x8BTgdOBcaXMR9jjClXkV5DrAdsKGb7sg7s2shfixTgQtykVelEdsuOMcZETaQ1xB9wNbmC/sT++xIjtU9EauKmDPhJVX/FNcOLuvG7Qtu9ezd33nkn/fr1Y9CgQXzySdFj5S5evJgbb7yR888/n8GDCw8fOXjwYPr27Uu/fv3o168fo0aNinbRK53atWvz9NNPs3jxYmbPns15551XZLoXX3yRRYsWhV7Lli3j3XffLZSuc+fOrFy5khEjRkS76JXKrl27GDVqFD169OD888/no48+KjLdwoULGTZsGGeeeSb9+/cvtP65555j0KBBnHrqqbzwwgvRLna5ibR2dw8wyY+YHQdcLCItgUuBP5Rxn1OA2UBN4Gm/rAOVsIb4xBNPULVqVd58803WrFnDbbfdxoknnkjjxo3zpUtISKBv37706tWLyZMnF5nX/fffT8eOHYModqV01113kZWVRbdu3TjppJMYP348K1asYM2aNfnSDR06NN/nV199lfnz5+dbFh8fz+23386SJUuiXu7K5uGHHyY+Pp4ZM2awatUqRowYQbNmzWjSpEm+dImJifTv35/evXszceLEQvk0aNCA4cOHM3369IBKXj4iqiGq6nu42mBv3DXEf+Jmyju/rIPEqupI3HPP16hqXkDMBUaWJZ9YS09PZ+7cuQwZMoTExEROPvlkunbtyqxZhYeLPOmkk+jduzd169aNQUkrv8TERHr37s0TTzxBWloa3377LbNnz2bAgJIHWa9fvz6dOnXi7bffzrd8yJAhfPnll6xbV9rATIeX9PR0Zs+ezbBhw0hKSqJdu3acfvrp/Pe//y2UtnXr1vTr14/69esXmdd5551Ht27dSEqqXH2lZRkgdgYwozx2qqozC3xeWB75BmnTpk3ExcXRoMH+2yqbNGnC0qVLDyi/Bx54gNzcXJo2bcqwYcMK/SIfzho1akROTg4pKSmhZStWrKBz584lbjdw4EAWLlzI5s2bQ8vq1avHH//4Ry688ELuvPPOErY+/Pz000/ExcVxwgknhJY1a9aMRYsWxbBUwYqohigi6/zk8gWXHyEiZfqZFZHGIjJFRJaLyE/hr7LkE2vp6emFfv2Sk5NJT49kjNz8br/9dqZMmcLrr79O+/btufnmm0lNTS2volZ6SUlJhc7Hnj17SE5OLnG7AQMG8NZbb+Vbdscdd4Rqmia/tLS0Que0Ro0ah9W5irRTpRHu2mFB1YGi68zFm4JrIv8duKzAq1gicrWILBSRhZMmTSrjLstfYmJioT+UtLQ0EhMTy5xXmzZtqF69OgkJCVx66aXUqFGD7777rryKWumlpaVRo0aNfMtq1KjB3r17i92mY8eO1KlThxkz9jdqevbsSXJyMh9++GHUylqZJSUlFTqne/furXTN3oNRYpNZRC4M+/gHEdkV9jkOOAtIKeM+WwPdyvjoH6r6AvACwObNmw9kPuhydfzxx5OTk8OmTZs4/nj3ZOLatWtp1KjRQectIrgpbAxASkpKqCm3YYO7+6tly5aFOlTCDRw4kFmzZuX70eratStt2rThiy++AKBmzZrk5OTQvHlzrr322ugeRCXQsGFDcnJy+Omnn2jYsCEAq1ev5sQTT4xxyYJTWg3xP/6lwMthn/8DTAJ64mp6ZfE50L6M21Q4iYmJ9OjRgwkTJpCens7333/PvHnzOOeccwqlzc3NJTMzk+zsbFSVzMxMsrKyANi2bRvff/89WVlZZGZmMnXqVHbt2kWbNm2CPqQKKz09nVmzZnH99deTmJhIhw4dOOuss3jnnXeKTF+9enXOPffcQs3lJ554gj59+jBw4EAGDhzI7NmzeeONN/jHP/4RxGFUeImJifTs2ZPx48eTnp7O0qVL+eyzz+jXr1+htLm5uWRkZIT+pjMyMkJ/0wDZ2dlkZGSgquTk5JCRkUFOTk6hfCqaEmuIqloFQETWA539PYMHKwX4SETeArYW2F+lGu3mhhtuYOzYsfzxj3+kVq1ajBgxgsaNG/Pdd99x6623hnrnvvvuO2688cbQdn379qVt27Y89thjpKWl8fjjj7NlyxaqVatGkyZNeOihh6hdu3asDqtCuueee3jggQeYN28eO3fu5O6772bNmjV07NiRF198kQ4dOoTSnn322ezevZuvv/46Xx579+7N1yTct28f6enp7Nq1C+Pccsst3HvvvfTu3ZvatWtz66230qRJExYvXswNN9zA559/Drh7a4cNGxbarnv37nTo0IHx490DZ/fddx8ffPBBaP2//vUv7rrrLs4///xgD6iMJOimmYhMKG5dpAM/VIQm8+GgV69esS7CYWPBggWxLsJho1atWlLcuohvuxGRI4FzcbPvVQtfp6rFjnFYkI12Y4ypqCId/utU4APcI3bHAJtx04lm4JrAEQdEn19L3NSmx6nqdX7g2eqqal2rxpiYifS2m7HAZNwtNvuAXria4kJgTFl2KCIX42bYqw9c7hfXJIIpTI0xJpoiDYi/B55Wd8ExB1eb2wbcAtxdxn2OBs5W1WHsn59lKdC2jPkYY0y5ijQgZoa93wbkPduTihsarCyOBfKaxhr2r3WUGGNiKtKAuAjIe3B0DnCfiPwVeJL9wS1S31L4qZRBuNG3jTEmZiLtZb4dd50P4A7clKRPAauAK8u4z+uBmSLyf0CyiMzAjbrdu4z5GGNMuYp0GtKFYe9/wd1+EzEROVJVd/jtV/he5vOA94GNwPuqaqMZGGNiKtLbbloDcQVvixGR3wPZqrq8lCw2ALX8Nh+r6tnAvw+gvMYYEzWRXkN8ASjq4dpWfl1p0kSkjYjEAaeIU6XgK9JCG2NMNER6DfH3FN3psQA4OYLt7/HbV/efswusF1wvc1FDjBljTCAiDYg5QFGjDRyJC2YlUtXnRORF4HfACtwQYMYYU6FEGhA/A24XkYtVNQdAROJxvc+fl7axiHytqqcCm0TkHVUtakpTY4yJqUgD4s3AF8AaEfnCL+sO1KDo6UkLai4iCaq6D6jY4/8YYw5bkd52s9L3KF8HtPOLJwPPquqWCLJ4B1glIilAoogUWatU1UiCqzHGREVZZt37GddELjNVHSIi3XFzs3QGXiKCa4/GGBOkiAPiwVLVL0Tka9xYin2AOsCvwCfAa6qaVdL2xhgTbYHd+ycitYF5wINAFu756Cz/eZ5fb4wxMRNYDREX+H4BeqpqaGILEakBTPPrbeozY0zMBPl0yEDgmvBgCOCfYf4bcEGAZTHGmELKFBBFpI6IdBGR6qWnLqQ2buqBomzCP+tsjDGxElFAFJGaIvJvYDvuOmB9v/x5Ebk7wn2txU09UJSzgHUR5mOMMVERaQ1xDC4IdgDSw5a/T+RN3UeBV0Xkj3kDOfhBHS4CJmJzqhhjYizSTpX+wAWqukREwof6/xE4MZIMVHWiiByNC36vi8ivuFtvMoDRqlrsfM3GGBOESAPikcBvRSyvyf6JokqlquNE5AXgNPbfh/iVqu6ONA9jjImWSAPiAlwt8XH/Oa+W+P9w1xQjpqp7gBll2cYYY4IQaUC8DZjhR86OB270708hssEdjDGmwouoU0VV5+GaudVwvcVnAVuArqq6KHrFM8aY4JRlcIdlwF+jWBZjjImpSCeZOqqk9ar6v/IpjjHGxE6kNcRf2d+RUhSbC8UYU+lFGhB7FvhcFWgPXIObuN4YYyq9SEfM/qyIxR+LyDrgKmBKuZbKGGNi4GBHu1mC3XZjjDlEHHBA9OMYjgA2ll9xjDEmdiLtZd5D/k4VAZKAvcCfo1AuY4wJXKSdKtcV+JyLG/16vqruKN8iGWNMbJQaEP2E9MnA2xFOOWqMMZVSqdcQVTUbGIu71cYYYw5ZkXaqfA10jGZBjDEm1kS1pAdQfCKRQcADwJPAt7jOlJAYDPBQeqHNQcvJiXioS3OQ4uODnADz8KaqUty6EgOiiPwLd2vNzpLz16Af3bOAGAALiMGxgBicgwmIOUBdILGUHWw44NIdGAuIAbCAGBwLiMEpKSCW9r8gPoOgA54xxgQukk4Vq40ZYw4LkdTTt4oUW8MEIAbXEI0xptxFEhCvpuROFWOMOSSU1qmSC/xOVbcHV6SIWDM+ANapEhzrVAlOSZ0qpV1DtMBjjDlslBYQS754aIwxh5AS6+mqerADyBpjTKVhAc8YYzwLiMYY41lANMYYzwKiMcZ4FhCNMcazgGiMMZ4FRGOM8SwgGmOMZwHRGGM8C4jGGONZQDTGGM8CojHGeBYQjTHGs4BojDGeBURjjPECDYgiEiciV4pI9SD3a4wxkShxTpWo7FBkp6oecZDZ2NQGAbA5VYJjc6oE52DmVImG90Tk/Bjs1xhjShSLn6UE4D8i8hWwkbDanqpeHoPyGGMMEJuA+L1/GWNMhRL4NcRyUikLXdnYNcTg2DXE4FS0a4iIyDki8rKIvOc/dxKRXrEoizHG5Ak8IIrIcOA5YDVwul+cDtwXdFmMMSZcLG67WQucpaopIrJDVY8UkThgu6oeHWE21mQOgDWZg2NN5uBUtCZzTVzvMuwPbFWBzBiUxRhjQmIRED8Hbi2w7Hrg0xiUxRhjQmLRZK4LvAfUAeoD64A9wHmqujXCbKzJHABrMgfHmszBKanJHJPbbkREgFOAhrjm8zeqmluGLCwgBsACYnAsIAanwgXE0M5F8jXZyxAULSAGwAJicCwgBqdCdaqISAcR+UpE9gJZ/pXt/61Udu7cyd/+9jfatWtHz549ee+994pMp6qMHTuWLl260KVLF8aOHUv4D1FOTg6PPfYY3bt3p3379gwcOJDdu3cHdRiVws6dOxk+fDgdO3bkrLPO4v333y8ynaoybtw4unbtSteuXRk3blzoXC9cuJCOHTvme7Vq1YqZM2cGeSgV2pFHHsn06dNJTU0lJSWFwYMHF5muWrVqPPfcc2zdupXffvuNd999l3r16uVLc8kll7B8+XJSU1NZs2YN3bt3D+IQDo6qBvoClgEPACcBJ4S/ypBPhTBy5Ei94YYbNDU1VRcsWKAdOnTQVatWFUr3+uuva+/evfXnn3/WrVu36rnnnqtTpkwJrX/00Uf1sssu002bNmlubq6uXLlS9+3bF+ShFCk7O7vCvEaMGKHXX3+97tq1S+fPn68dOnTQH3/8sVC6yZMna+/evXXTpk26efNm7du3r06aNKnIPOfNm6ft2rXT3bt3x/z4cK2emL+mTJmiU6dO1eTkZO3WrZvu3LlTW7VqVSjdqFGjdMmSJXrsscdq9erV9ZVXXtE333wztP7ss8/WlJQU7dKli4qI1qtXT+vVqxfz43Mhr4T4VNLKaLyA3fim+kG8Ym7v3r3aunVrXbduXWjZTTfdpGPHji2U9pJLLtGpU6eGPv/73//Wiy++WFVVd+7cqe3atdMNGzZEv9BlFOsgkffavXu3tmrVStesWRNa9ve//10ffvjhQmn/9Kc/6ZQpU0Kfp02bphdddFGR+d5888168803x/z4KkpATEpK0oyMDG3WrFlo2auvvqoPPvhgobTPPvusjijWkRoAABQ2SURBVBkzJvS5X79+umLFitDnL7/8Uq+88sqYH1NZA2Isbrt5C+gdg/2Wq5SUFOLi4mjcuHFoWcuWLVmzZk2htKtXr6Zly5b50q1evRqAVatWERcXx0cffUS3bt3o06cPkydPjv4BVCIpKSnEx8fTqFGj0LIWLVoUea7XrFlDixYtSk2XlpbGzJkzGThwYFTKXBk1b96c7Ozs0N8mwNKlS2ndunWhtC+//DLdunWjbt26JCYm8uc//5kPP/wQgCpVqtCpUyeOOeYYVq9ezcaNG3nqqadISEgI7FgOVCBXckXkNVx0BqgOvCUiXwD5brPRSjT8V1paGjVq1Mi3rGbNmuzdu7fUtDVr1iQtLQ1VZevWrezZs4eUlBQ++eQTUlJSuOKKK2jUqBHdunWL+nFUBmlpaSQnJ+dbVtK5rlmzZuhzjRo1Qufa3dzgfPzxxxx55JF07tw5egWvZGrUqFHo2vWuXbvync88eYFuy5YtZGdns2zZMq677joAjjvuOKpVq8ZFF11Ejx49yMrK4p133uGOO+7gjjvuCORYDlRQNcQ1wFr/Wg6MAb4MW5b3qjSSkpJITU3Ntyw1NbXQFzcvbfiXNzU1laSkJEQk9Kv5t7/9jYSEBFq2bMkf/vAHPvvss+geQCVS8PxByec6/P9l7969oXMd7u2336Z///6Flh/OUlNTqVWrVr5ltWrVYs+ePYXSPvPMM1SvXp2jjjqK5ORkpk+fHqohpqenA/DUU0+FOl0effRR+vXrF/2DOEiBBERVvSfvBYwP/xy+vKQ8RORqEVkoIgtfeOGFIIpdokaNGpGTk0NKSkpo2YoVK2jatGmhtM2aNWPFihX50jVr1gwg1LwL/2LalzS/Ro0akZ2dne9cr1y5sshz3bRpU1auXBn6XNT/yc8//8yCBQsYMGBA1MpcGa1atYr4+Ph856tt27b88MMPhdK2a9eOiRMnsmPHDjIzM3nqqafo0qULRx99NDt37mTjxo357qQIf1+RxeIa4qpili8vaSNVfUFVO6lqp6uvvjoKxSqbpKQkzjnnHJ588knS0tL49ttv+eSTT4r8kg0YMIAJEyawbds2tm3bxoQJE7jgggsAaNiwIZ06deL5558nMzOTtWvX8sEHH9CzZ8+gD6nCyjvXTz/9NGlpaSxatIjZs2dz/vmFZ6Lo378/r7zyCtu2bWP79u1MnDix0HXCd999l3bt2tGwYcOgDqFSSEtLY/r06YwePZqkpCROO+00BgwYwGuvvVYo7YIFC7j88supVasW8fHxXHvttWzevJnffvsNgAkTJjB8+HCOOeYYjjjiCEaOHFnsrVIVSkk9LtF4AXuKWFYL+LUM+VQIO3bs0GuuuUbbtm2rZ5xxhr777ruqqrpgwQJt165dKF1ubq6OGTNGO3furJ07d9YxY8Zobm5uaP3WrVv1yiuv1Hbt2mmvXr309ddfD/xYihLrntfw16+//qrDhg3Ttm3b6umnn65vv/22Zmdn69dff61t27YNpcvKytKHHnpIO3XqpJ06ddKHHnpIs7Ky8uXVp08fnTZtWsyPqaL1MgN65JFH6ltvvaWpqam6YcMGHTx4sALavXt33bNnTyjdUUcdpZMmTdJt27bpjh07dO7cudq5c+fQ+vj4eH3mmWd0x44d+vPPP+sTTzyh1atXj/nxUUovc2BPqohI3vwp9YAtBVYfDbyuqldFmF3lqH9XcvakSnDsSZXgaEV4dE9EzgAE+C9wbtgqBbap6soiNyyaBcQAWEAMjgXE4FSIgBjaoUiSqqYdZDYWEANgATE4FhCDU1JADOo+xNtV9X7/8dbielFV9a4gymOMMUUJ6mfp+LD3DQLapzHGlIlNQ2qKZU3m4FiTOTgxbzIXJCItgYuB41T1OhFpAVRX1e9iUR5jjIHYjId4MTAXN31A3rPLNYFHgy6LMcaEi0Uv84/AIFVdGjYNaVVgi6oeE2E21mQOgDWZg2NN5uCU1GSOxaN7xwJ5TWMN+9eCnDEmpmIREL8FLiuwbBDwTQzKYowxIbFoMrcEZgLrgVOBOUAL4BxVXV3CpuGsNhkAazIHx5rMwalQT6qAe1oFOA83l8pPwAeqmlryVvlYQAyABcTgWEAMToUIiCLyKSUHMlXVsyLMzgJiACwgBscCYnAqyn2Ik4pZXh+4HkgKsCzGGFNIzJ5UEZGjgX8AQ4FpwGhV3RTh5lZDDIDVEINjNcTgVKjbbkSklojci5tn5Tigg6peXYZgaIwxURFYQBSRRBH5B7AON0l9d1W9TFUr1eRSxphDV5CdKttwAXgssLCoNKo6O8LsrMkcAGsyB8eazMGpKL3MKZTey3xihNlZQAyABcTgWEAMToUIiOWsUha6srGAGBwLiMGpUJ0qxhhTUVlANMYYzwKiMcZ4FhCNMcazgGiMMZ4FRGOM8SwgGmOMZwHRGGM8C4jGGONZQDTGGM8CojHGeBYQjTHGs4BojDGeBURjjPEsIBpjjGcB0RhjPAuIxhjjWUA0xhjPAqIxxngWEI0xxrOAaIwxXmWdda9SEpGrVfWFWJfjUGfnOTiH2rm2GmKwro51AQ4Tdp6Dc0idawuIxhjjWUA0xhjPAmKwDplrLRWcnefgHFLn2jpVjDHGsxqiMcZ4FhBNhSDOBBHZISLf+GXXiMg2EUkVkaOjtN/nReTOaOR9KCntPInI3SIyKcgyRYMFxAMgIikicnaBZVeIyBcRbBtRukORP/ZlIpImIltF5DkROcKv7g6cAxyvqqeISFXgUaC3qtZQ1d+iUSZVHaaq90Yj74rG/91uF5HksGVXicic0rYNP08icqaIbIpiUWPGAqIJhIj8HRgDjAJqA6cCJwCzRKSaf5+iqnv9JscBCcAPMSjuoSwOuCHWhSiJiMTHat8WEKNARG4VkbUiskdElovIBX75ScDzQFffDNzpl1cXkUdE5CffRHxeRBJjeQzlSURqAfcAw1X1I1XNUtUU4E9AI+Ay4CX2n5fXgZV+850iMtvn01JEZonI/0RkpYj8KWwfE0XkGRH5wJ/3+SLSxK8TEXnM1452+1pqm7Dt7vPvfxSR88LyjBeRX0Skg/98qojME5GdIrJURM6M5nmLkrHATWE185AIzu99vnb5IVDP/1+likg9n6yaiLzqz/8PItIpbPt6IvKmP5/rReT6sHV3i8h/RGSSiOwGrojWwZfGAmJ0rAV64GpC9wCTRKSuqv4IDAO+8s3AvD/Kh4DmQDugKVAfuCv4YkfNabja3vTwhaqaCvwXOJv852Uw0NonO0JVe/kv4ixgCnAsMAh4VkRahWU5CHe+jwTWAPf75b2B03HnuDYuEBfVBH8dGBz2uQ/wq6ouEpH6wAfAfcBRwE3AmyJyTBnPRawtBObgyh8S4fnF1+DPBbb4/6saqrrFr+4PTAWOAN4FnvZ5VwHeA5bi/rbPAkaISJ+wrAcA//HbTi6vgy0rC4gH7m1fU9jpa3rP5q1Q1TdUdYuq5qrqNGA1cEpRmYiI4B5/Gqmq/1PVPcADuD/IQ0UdXGDJLmLdz359ac7DNaknqGq2qi4G3gQuDkvzlqp+4/czGfcDA5AF1ARa4m41+1FVfy5iH1OA/iKS5D9figuSAH8B/quq//X/r7NwwaVfBGWvaO4ChhcI5pGc39J84c9PDvAa0NYv7wwco6qjVTVTVdcBL5L/b/wrVX3bn9v0Az6ygxSztvohYKCqfpz3QUSuAK7y7y8HbsQ1BwFqUPyX/hggCfjWxUaXHe5az6HiV6COiMQXERTr+vWlOQHokneZwYvHffHybA17n4Y776jqbBF5GngGOEFEpgM3qeru8B2o6hoR+RE4X0Tew9V42oft/2IROT9sk6rApxGUvUJR1e9F5H3gVuBHvziS81uaguc/wV8PPAHXxA7POw6YG/Z5Yxn2EzUWEMuZiJyA+/U7C/erlyMiS3BBDqDgnfC/AulAa1XdHFxJA/UVkAFcCPw7b6GI1MA1v26LII+NwGeqes6BFEBVnwSeFJFjfRlGAUXdRpLXbK4CLFfVNWH7f01Vhx7I/iugfwKLgHH+c1nOb1mf5tgIrFfVZuWYZ1RYk7n8JeP+c38BEJEhQJuw9duA433PKqqaiwugj/kvKyJSv8D1lUpNVXfhru09JSJ9RaSqiDTCBaZNRFYLeR9oLiKX+e2rikhn31FVIp+ui7hbefYC+4DcYpJPxV1zvAbXhM4zCVdz7CMicSKSIO72k+MjKHuF4wP9NCCvc6Ms53cbcLSI1I5wd98Ae0TkFhFJ9OevjYh0PvgjKV8WEMuZqi7H/ep+hfvDORn4MizJbNytJFtFJK+peAuuE+Br38v2MdAisEIHQFUfxtUEHwF2A/NxNYezVDUjgu334ALVIGALrnk2Bqgewe5r4X50dgAbcB0qY4vZz8+4/7vTcAEjb/lG3IX/23A/dhtxtczK/B0ajfsBL9P5VdUVuJr0On8NvV7BNAXS5+CuUbYD1uNaRS/hOrgqFHuW2RhjvMr862aMMeXKAqIxxngWEI0xxrOAaIwxngVEY4zxLCAaY4xnAdGUSkQuEhEN+3yFiKTGqCzvi8jEWOw7lkRkjn/80ESRBcRKyg/HpP6VJSLrxA0hllz61gdtGnBipInFDUx6U+kpY09EGvlz2qn01IG6EPhHrAtxqLNnmSu3j3FjCVbFDTf2Eu7Jg2sKJvQP2edoOdyJ70cjidmIJIcTEanmR4j5X6zLcjiwGmLllqGqW1V1o6pOwQ15NRBCg25+75u3a3GDKySLSG0ReUHcYKl7ROSzgrUhEblcRDaIG+r/fdzo1eHrCzWZRaSfuEFZ00XkNxF5zz/vOwc32snYvBpt2Dan+f2nichmcVMK1Apbn+RrwqniBs6NZBCIvIFcZ4vIXhHZ5d/X8+v6ishccXO3/E9EZhR4Xne9/3eBL++csHyHiBvwd5+IrBKRkeLG+stb39wfzz5xA6z282W/IizNySLysT9P//PHVzts/UR/WeAWccP0b/LL8zWZRaSaiIwRkU3+/C2QsOff/bPIT4rIFhHJEJGNIvJQJOfvcGYB8dCSjqst5mmMG9PvYtzYdBm4QU7r454tbQ98DswWkboAItIFmIibb7cdbmDP0SXtVET64gYEnQV0BHoCn+H+vi7EfalH44b6ytvPycBMv11bn64d8K+wrB/BzbPyR9zoQe1xA72WVJa2uCG51gDdcFMVTGN/aygZeBw3PuWZwC7gPfGDbbB/3Mq+vqwX+nyH4sapvAs4Cfg77hn0a/36KsBbQLbf5xW4EWVCzwL7yxkzgFS/nwtwz0yHHzPAGcDvfRnOKuZQJ/h0l+IGD3nFH0feGITX+/wHAc2AS9g/CrkpjqraqxK+cEHr/bDPp+Aemp/mP9+NGxj1uLA0vXBfxsQCeS0BbvbvpwCzCqx/yf2phD5fAaSGff4SmFpCWVNw4w+GL3sVeLnAsna4kYKOxY1lmAH8OWx9DWAnMLGEfU3GDbsW6XlMBnKA7v5zI1+GTgXS/QRcVmDZCNwQYeBG184G6oetP83ndYX/PBQXgGuGpTnTp2ka9v/6C1C9wL7mAE/7901wo/U0LJDmbeBZ//5J4BP8eAX2iuxl1xArt76+6RqPqxm+AwwPW79JVbeFfe6IG4z2F9k/GC244f2b+Pcn4WqF4b4C/q+EcrTHfZHLoiPQVEQuCVuWV6gmuAFGq/l9A27KARFZVkq+7XE1tSKJm2flXqALbnDeKv7VsIRtjgEaAONF5LmwVfFhZW6JG1Y/fEzLBeQfZuwk4Dt1I8vkmefTtMLVagG+15JHAOrg97u8wP9jddxoSuD+P2YBq0RkJm6qhg/VDTdnimEBsXL7HDf9QBbuy5hVYP3eAp+r4IYk61FEXruLWBZNVXA1z8eKWLcZN/9JNLyPa8L/P7+fbGA5LvgWJ+/S0jBcAIuG8M6ugv9vBVXx6Tvj/u/DpQOomwemEa7mehauSb1URM6xoFg8C4iVW5ruH9E5EotwHSS56ua1KMqPuGtg4Qp+Lmgx7kv3YjHrMyk8JcIi3CjhRZbfdwRl+X2v88uScdfL1pZSll7F5Hk0riZ3rap+6pd1IP/3INP/Gyqvqm4TkS1AE1V9tZj9rsANk19P90+61In81+l/BK4UkZphtcTTfJofidxiXA3xd3nHURS/j/8A/xF37+bXuEnMVpVhX4cV61Q5vHyMu973joicKyKNRaSriNwjInm1xieBs0XkHyLSzHcmXFBKvvfj5hu5T0RaiUhr3wObN1lTCtBD3EjgeXPLjAFOETflansRaSoi54nIeAjNyPcyMEZEzhGR1rjOh9LmmhkLtBfXk95WRFqIm4y9IW6A2F+BoX5/Z+CmhQ2f52U7rpbVR0SOC+sB/idwsz+uFuJGfL5cRPLuDZyF67R4xe/3VOBRn3de7W8y7lLAq763+XRgPDC9LD9sqrrK5zVR3E3zJ4pIJxG5SUTyOoFuFJHBInKSiDTFdb7sxvdam2LE+iKmvQ7sRYFOlSLW3427FlVweU3gCdwXIxM38vNUXO0nL80QXCdCOm4O3usooVPFL+sPfIvrCPkV13uc4NedipuCcl+BfDoBH+G+qHuBZcDosPXJuM6XVFyguhPX5J1YyrnpjruckI7rhPkYqOvX9QK+92X5HtekTMV3fPg0V/njzwHmhC0fjKvZ7sMF1y+AQWHrm/v9ZuCC43n+HF8SluZkXGdHus9jIlC7tP9XwjpV/Oeq/v94nd/HVn/OO/r1Q31Z9/jz+xlwWqz/biv6y0bMNiZK/C0wS3A91t/GujymdBYQjSknInIBrqa7Gnf7zqO4a33t1b5olYJ1qhhTfmriro02wDWH5wAjLRhWHlZDNMYYz3qZjTHGs4BojDGeBURjjPEsIBpjjGcB0RhjPAuIxhjj/X83rpN4F2vmwAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "# Plot confusion matrix\n",
        "plot_confusion_matrix(y_test,y_preds)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "1y-3EyrAW6sH",
        "outputId": "a288e40d-73c1-4a69-d475-8ae4f4db397b"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQIElEQVR4nO3dfYxldX3H8fen7IpViaA7jQRWBiKJgUYEJ4gPaVFr5EHdNtJkSatgMVuttJqaJigJtvxT/EcbxUg2QgrGIBYfsuoSuy00ag2rw2Z5Fl2QFggpIyBItNg13/5xz+rlOjP3zs69d8af71dys+ec3++e8+V3D585cx7upKqQJP3m+521LkCSNB4GuiQ1wkCXpEYY6JLUCANdkhqxYa02vGnTppqdnV2rzUvSb6RbbrnlR1U1s1jbmgX67Ows8/Pza7V5SfqNlOS/lmrzlIskNcJAl6RGGOiS1AgDXZIaYaBLUiOGBnqSZyf5TpJbk9yZ5B8W6XNokuuS7EuyO8nsJIqVJC1tlCP0p4HXV9VJwMuBM5KcNtDnAuDxqnoJ8DHgI+MtU5I0zNBAr56nutmN3WvwO3e3AFd309cDb0iSsVUpSRpqpHPoSQ5Jshd4BNhVVbsHuhwFPABQVfuBJ4AXLrKebUnmk8wvLCysrnJJ0jOM9KRoVf0CeHmSw4EvJfn9qrpjpRurqu3AdoC5uTn/sobWrdmLvrYm273/srPXZLtqw4rucqmqHwM3AWcMND0EbAZIsgF4PvDoOAqUJI1mlLtcZrojc5L8LvBG4HsD3XYA53XT5wA3ln/bTpKmapRTLkcCVyc5hN4PgM9X1VeTXArMV9UO4ErgM0n2AY8BWydWsSRpUUMDvapuA05eZPklfdP/C/zpeEuTJK2ET4pKUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqxNBAT7I5yU1J7kpyZ5L3LdLn9CRPJNnbvS6ZTLmSpKVsGKHPfuADVbUnyWHALUl2VdVdA/2+WVVvHn+JkqRRDD1Cr6qHq2pPN/0T4G7gqEkXJklamRWdQ08yC5wM7F6k+VVJbk1yQ5ITl3j/tiTzSeYXFhZWXKwkaWkjB3qS5wFfAN5fVU8ONO8Bjqmqk4BPAF9ebB1Vtb2q5qpqbmZm5mBrliQtYqRAT7KRXph/tqq+ONheVU9W1VPd9E5gY5JNY61UkrSsUe5yCXAlcHdVfXSJPi/q+pHk1G69j46zUEnS8ka5y+U1wNuB25Ps7ZZ9CHgxQFVdAZwDvCfJfuBnwNaqqgnUK0lawtBAr6pvARnS53Lg8nEVJUlaOZ8UlaRGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGDA30JJuT3JTkriR3JnnfIn2S5ONJ9iW5LckpkylXkrSUDSP02Q98oKr2JDkMuCXJrqq6q6/PmcDx3euVwKe6fyVJUzL0CL2qHq6qPd30T4C7gaMGum0Brqmem4HDkxw59molSUta0Tn0JLPAycDugaajgAf65h/k10OfJNuSzCeZX1hYWFmlkqRljRzoSZ4HfAF4f1U9eTAbq6rtVTVXVXMzMzMHswpJ0hJGCvQkG+mF+Wer6ouLdHkI2Nw3f3S3TJI0JaPc5RLgSuDuqvroEt12AO/o7nY5DXiiqh4eY52SpCFGucvlNcDbgduT7O2WfQh4MUBVXQHsBM4C9gE/Bd45/lIlScsZGuhV9S0gQ/oU8N5xFSVJWjmfFJWkRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktSIoYGe5KokjyS5Y4n205M8kWRv97pk/GVKkobZMEKffwYuB65Zps83q+rNY6lIknRQhh6hV9U3gMemUIskaRXGdQ79VUluTXJDkhOX6pRkW5L5JPMLCwtj2rQkCcYT6HuAY6rqJOATwJeX6lhV26tqrqrmZmZmxrBpSdIBqw70qnqyqp7qpncCG5NsWnVlkqQVWXWgJ3lRknTTp3brfHS165UkrczQu1ySXAucDmxK8iDwYWAjQFVdAZwDvCfJfuBnwNaqqolVLEla1NBAr6pzh7RfTu+2RknSGvJJUUlqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSI4b+gQtJatHsRV9bs23ff9nZE1mvR+iS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRgwN9CRXJXkkyR1LtCfJx5PsS3JbklPGX6YkaZhRjtD/GThjmfYzgeO71zbgU6svS5K0UkMDvaq+ATy2TJctwDXVczNweJIjx1WgJGk04ziHfhTwQN/8g90ySdIUTfWiaJJtSeaTzC8sLExz05LUvHEE+kPA5r75o7tlv6aqtlfVXFXNzczMjGHTkqQDxhHoO4B3dHe7nAY8UVUPj2G9kqQVGPp96EmuBU4HNiV5EPgwsBGgqq4AdgJnAfuAnwLvnFSxkqSlDQ30qjp3SHsB7x1bRZKkg+KTopLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNWLofejr0exFX1uzbd9/2dlrtm1JWo5H6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGjFSoCc5I8k9SfYluWiR9vOTLCTZ273eNf5SJUnLGfo3RZMcAnwSeCPwIPDdJDuq6q6BrtdV1YUTqFGSNIJRjtBPBfZV1X1V9XPgc8CWyZYlSVqpUQL9KOCBvvkHu2WD3pbktiTXJ9m82IqSbEsyn2R+YWHhIMqVJC1lXBdFvwLMVtXLgF3A1Yt1qqrtVTVXVXMzMzNj2rQkCUYL9IeA/iPuo7tlv1RVj1bV093sp4FXjKc8SdKoRgn07wLHJzk2ybOArcCO/g5JjuybfStw9/hKlCSNYuhdLlW1P8mFwNeBQ4CrqurOJJcC81W1A/ibJG8F9gOPAedPsGZJ0iKGBjpAVe0Edg4su6Rv+oPAB8dbmiRpJXxSVJIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiNGCvQkZyS5J8m+JBct0n5okuu69t1JZsddqCRpeUMDPckhwCeBM4ETgHOTnDDQ7QLg8ap6CfAx4CPjLlSStLxRjtBPBfZV1X1V9XPgc8CWgT5bgKu76euBNyTJ+MqUJA2zYYQ+RwEP9M0/CLxyqT5VtT/JE8ALgR/1d0qyDdjWzT6V5J6DKRrYNLjuacnyv3usWV0jWK+1WVefIfsXOF4rtS7rykdWVdcxSzWMEuhjU1Xbge2rXU+S+aqaG0NJY7Ve64L1W5t1rYx1rcxvW12jnHJ5CNjcN390t2zRPkk2AM8HHh1HgZKk0YwS6N8Fjk9ybJJnAVuBHQN9dgDnddPnADdWVY2vTEnSMENPuXTnxC8Evg4cAlxVVXcmuRSYr6odwJXAZ5LsAx6jF/qTtOrTNhOyXuuC9Vubda2Mda3Mb1Vd8UBaktrgk6KS1AgDXZIase4CfTVfM5Dkg93ye5K8acp1/W2Su5LcluTfkxzT1/aLJHu71+AF5UnXdX6Shb7tv6uv7bwkP+he5w2+d8J1fayvpu8n+XFf2yTH66okjyS5Y4n2JPl4V/dtSU7pa5vkeA2r68+6em5P8u0kJ/W13d8t35tkfsp1nZ7kib7P65K+tmX3gQnX9Xd9Nd3R7VMv6NomMl5JNie5qcuBO5O8b5E+k92/qmrdvOhddL0XOA54FnArcMJAn78CruimtwLXddMndP0PBY7t1nPIFOt6HfCcbvo9B+rq5p9aw/E6H7h8kfe+ALiv+/eIbvqIadU10P+v6V1sn+h4dev+A+AU4I4l2s8CbgACnAbsnvR4jVjXqw9sj97XcOzua7sf2LRG43U68NXV7gPjrmug71vo3Xk30fECjgRO6aYPA76/yP+PE92/1tsR+mq+ZmAL8Lmqerqqfgjs69Y3lbqq6qaq+mk3ezO9+/UnbZTxWsqbgF1V9VhVPQ7sAs5Yo7rOBa4d07aXVVXfoHcn1lK2ANdUz83A4UmOZLLjNbSuqvp2t12Y3v41yngtZTX75rjrmsr+VVUPV9WebvonwN30nqLvN9H9a70F+mJfMzA4IM/4mgHgwNcMjPLeSdbV7wJ6P4UPeHaS+SQ3J/njMdW0krre1v16d32SAw+JrYvx6k5NHQvc2Ld4UuM1iqVqn+R4rdTg/lXAvya5Jb2v15i2VyW5NckNSU7slq2L8UryHHrB+IW+xRMfr/ROBZ8M7B5omuj+NdVH/38bJPlzYA74w77Fx1TVQ0mOA25McntV3Tulkr4CXFtVTyf5S3q/3bx+StsexVbg+qr6Rd+ytRyvdS3J6+gF+mv7Fr+2G6/fA3Yl+V53BDsNe+h9Xk8lOQv4MnD8lLY9ircA/1lV/UfzEx2vJM+j9wPk/VX15LjWO4r1doS+mq8ZGOW9k6yLJH8EXAy8taqePrC8qh7q/r0P+A96P7mnUldVPdpXy6eBV4z63knW1WcrA78OT3C8RrFU7ZMcr5EkeRm9z3BLVf3yqzX6xusR4EuM71TjUFX1ZFU91U3vBDYm2cQ6GK/OcvvX2McryUZ6Yf7ZqvriIl0mu3+N+8LAKi8qbKB3MeBYfnUh5cSBPu/lmRdFP99Nn8gzL4rex/guio5S18n0LgIdP7D8CODQbnoT8APGdHFoxLqO7Jv+E+Dm+tVFmB929R3RTb9gWnV1/V5K7wJVpjFefduYZemLfGfzzItW35n0eI1Y14vpXRd69cDy5wKH9U1/GzhjinW96MDnRy8Y/7sbu5H2gUnV1bU/n9559udOY7y6/+5rgH9aps9E96+xDe4YP6Sz6F0dvhe4uFt2Kb2jXoBnA//S7dzfAY7re+/F3fvuAc6ccl3/BvwPsLd77eiWvxq4vduhbwcumHJd/wjc2W3/JuClfe/9i24c9wHvnGZd3fzfA5cNvG/S43Ut8DDwf/TOU14AvBt4d9ceen/Q5d5u+3NTGq9hdX0aeLxv/5rvlh/XjdWt3ed88ZTrurBv/7qZvh84i+0D06qr63M+vRsl+t83sfGidxqsgNv6Pqezprl/+ei/JDVivZ1DlyQdJANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNeL/AUVX2UA+sASKAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "# Plot distribution of estimates. Set the density parameter to True to normalize bin heights so that the integral of the histogram is 1.\n",
        "# The resulting histogram is an approximation of the probability density function. This facilitates comparison between the distribution \n",
        "# of estimates and true distribution.\n",
        "plt.hist(y_preds, density=True)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 266
        },
        "id": "RTtoVuOVcvy9",
        "outputId": "b0a85099-2741-43ff-ebe2-c56b068049cb"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD5CAYAAAA3Os7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASIElEQVR4nO3df6zldX3n8edrhxHb6oo6t3ECM1yMJI00VvCG9UezS3VNESyzGzEds23BxczWylazzW6gJnTLP6v/1MbQlEyAFKxBXHTNqEO6swvGuobRCxmGX9KOlC1MSLkFBCdWuuO+94/zpT0cz73ne+eec+bux+cjOZnv9/v5nO/3zed+ed3v/f44J1WFJOn/f//kZBcgSZoOA12SGmGgS1IjDHRJaoSBLkmNMNAlqRGn9O2YZAuwDBytqveOtJ0K3AK8BXga+NWqemyt9W3btq0WFxfXW68k/US75557/raqFsa19Q504KPAw8A/HdN2BfBsVb0hyW7gk8CvrrWyxcVFlpeX17F5SVKS/71aW69TLknOAC4Gblilyy7g5m76duBdSbKeIiVJG9P3HPofAv8J+L+rtJ8OPA5QVceB54DXbrg6SVJvEwM9yXuBp6rqno1uLMmeJMtJlldWVja6OknSkD5H6O8ALknyGPA54J1J/nSkz1FgB0CSU4BXMbg4+hJVtbeqlqpqaWFh7Dl9SdIJmhjoVXV1VZ1RVYvAbuDOqvq1kW77gMu66Uu7Pn7qlyTN0XrucnmJJNcCy1W1D7gR+EySI8AzDIJfkjRH6wr0qvoa8LVu+pqh5T8E3j/NwiRJ6+OTopLUCANdkhpxwufQpZYtXvXVk7Ldxz5x8UnZrtrgEbokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNmBjoSV6e5FtJ7kvyYJLfH9Pn8iQrSQ51rw/NplxJ0mr6fMHFC8A7q+pYkq3AN5LcUVV3j/S7raqunH6JkqQ+JgZ6VRVwrJvd2r1qlkVJktav1zn0JFuSHAKeAg5U1cEx3d6X5HCS25PsmGqVkqSJegV6Vf2oqt4MnAGcn+TnR7p8GVisqjcBB4Cbx60nyZ4ky0mWV1ZWNlK3JGnEuu5yqarvAXcBF44sf7qqXuhmbwDessr791bVUlUtLSwsnEi9kqRV9LnLZSHJad30TwHvBr4z0mf70OwlwMPTLFKSNFmfu1y2Azcn2cLgF8Dnq+orSa4FlqtqH/DbSS4BjgPPAJfPqmBJ0nh97nI5DJw7Zvk1Q9NXA1dPtzRJ0nr4pKgkNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEb0+ZLolyf5VpL7kjyY5PfH9Dk1yW1JjiQ5mGRxFsVKklbX5wj9BeCdVfULwJuBC5O8daTPFcCzVfUG4FPAJ6dbpiRpkomBXgPHutmt3atGuu0Cbu6mbwfelSRTq1KSNFGvc+hJtiQ5BDwFHKiqgyNdTgceB6iq48BzwGvHrGdPkuUkyysrKxurXJL0Er0Cvap+VFVvBs4Azk/y8yeysaraW1VLVbW0sLBwIquQJK1iXXe5VNX3gLuAC0eajgI7AJKcArwKeHoaBUqS+ulzl8tCktO66Z8C3g18Z6TbPuCybvpS4M6qGj3PLkmaoVN69NkO3JxkC4NfAJ+vqq8kuRZYrqp9wI3AZ5IcAZ4Bds+sYknSWBMDvaoOA+eOWX7N0PQPgfdPtzRJ0nr4pKgkNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1os93iu5IcleSh5I8mOSjY/pckOS5JIe61zXj1iVJmp0+3yl6HPidqro3ySuBe5IcqKqHRvr9eVW9d/olSpL6mHiEXlVPVtW93fT3gYeB02ddmCRpfdZ1Dj3JIoMvjD44pvltSe5LckeSc6ZQmyRpHfqccgEgySuALwAfq6rnR5rvBc6sqmNJLgK+BJw9Zh17gD0AO3fuPOGiJUk/rtcRepKtDML8s1X1xdH2qnq+qo510/uBrUm2jem3t6qWqmppYWFhg6VLkob1ucslwI3Aw1X1B6v0eV3XjyTnd+t9epqFSpLW1ueUyzuAXwfuT3KoW/a7wE6AqroeuBT4cJLjwN8Bu6uqZlCvJGkVEwO9qr4BZEKf64DrplWUJGn9fFJUkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1Ij+nxJ9I4kdyV5KMmDST46pk+SfDrJkSSHk5w3m3IlSavp8yXRx4Hfqap7k7wSuCfJgap6aKjPe4Czu9c/A/64+1eSNCcTj9Cr6smqureb/j7wMHD6SLddwC01cDdwWpLtU69WkrSqdZ1DT7IInAscHGk6HXh8aP4Jfjz0JUkz1DvQk7wC+ALwsap6/kQ2lmRPkuUkyysrKyeyCknSKnoFepKtDML8s1X1xTFdjgI7hubP6Ja9RFXtraqlqlpaWFg4kXolSavoc5dLgBuBh6vqD1bptg/4je5ul7cCz1XVk1OsU5I0QZ+7XN4B/Dpwf5JD3bLfBXYCVNX1wH7gIuAI8APgg9MvVZK0lomBXlXfADKhTwEfmVZRkqT180lRSWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmN6PMl0TcleSrJA6u0X5DkuSSHutc10y9TkjRJny+J/hPgOuCWNfr8eVW9dyoVSZJOyMQj9Kr6OvDMHGqRJG3AtM6hvy3JfUnuSHLOlNYpSVqHPqdcJrkXOLOqjiW5CPgScPa4jkn2AHsAdu7cOYVNS5JetOEj9Kp6vqqOddP7ga1Jtq3Sd29VLVXV0sLCwkY3LUkasuFAT/K6JOmmz+/W+fRG1ytJWp+Jp1yS3ApcAGxL8gTwe8BWgKq6HrgU+HCS48DfAburqmZWsSRprImBXlUfmNB+HYPbGiVJJ5FPikpSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJasTEQE9yU5KnkjywSnuSfDrJkSSHk5w3/TIlSZP0OUL/E+DCNdrfA5zdvfYAf7zxsiRJ6zUx0Kvq68Aza3TZBdxSA3cDpyXZPq0CJUn9TOMc+unA40PzT3TLfkySPUmWkyyvrKxMYdOSpBfN9aJoVe2tqqWqWlpYWJjnpiWpedMI9KPAjqH5M7plkqQ5mkag7wN+o7vb5a3Ac1X15BTWK0lah1MmdUhyK3ABsC3JE8DvAVsBqup6YD9wEXAE+AHwwVkVK0la3cRAr6oPTGgv4CNTq0iSdEJ8UlSSGmGgS1IjDHRJasTEc+iS1KLFq7560rb92Ccunsl6PUKXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqRK9AT3JhkkeSHEly1Zj2y5OsJDnUvT40/VIlSWvp852iW4A/At4NPAF8O8m+qnpopOttVXXlDGqUJPXQ5wj9fOBIVT1aVX8PfA7YNduyJEnr1SfQTwceH5p/ols26n1JDie5PcmOqVQnSeptWhdFvwwsVtWbgAPAzeM6JdmTZDnJ8srKypQ2LUmCfoF+FBg+4j6jW/YPqurpqnqhm70BeMu4FVXV3qpaqqqlhYWFE6lXkrSKPoH+beDsJGcleRmwG9g33CHJ9qHZS4CHp1eiJKmPiXe5VNXxJFcCfwZsAW6qqgeTXAssV9U+4LeTXAIcB54BLp9hzZKkMSYGOkBV7Qf2jyy7Zmj6auDq6ZYmSVoPnxSVpEYY6JLUCANdkhphoEtSIwx0SWqEgS5Jjeh12+Jms3jVV0/ath/7xMUnbduStBaP0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEb0CvQkFyZ5JMmRJFeNaT81yW1d+8Eki9MuVJK0tomBnmQL8EfAe4A3Ah9I8saRblcAz1bVG4BPAZ+cdqGSpLX1OUI/HzhSVY9W1d8DnwN2jfTZBdzcTd8OvCtJplemJGmSPoF+OvD40PwT3bKxfarqOPAc8NppFChJ6meun4eeZA+wp5s9luSRE1zVNuBvp1PV+mTtk0knra4eNmtt1jVkwv4Fjtd6bcq68skN1XXmag19Av0osGNo/oxu2bg+TyQ5BXgV8PToiqpqL7C3xzbXlGS5qpY2up5p26x1weatzbrWx7rW5yetrj6nXL4NnJ3krCQvA3YD+0b67AMu66YvBe6sqppemZKkSSYeoVfV8SRXAn8GbAFuqqoHk1wLLFfVPuBG4DNJjgDPMAh9SdIc9TqHXlX7gf0jy64Zmv4h8P7plramDZ+2mZHNWhds3tqsa32sa31+ouqKZ0YkqQ0++i9Jjdh0gb6RjxlIcnW3/JEkvzznuv5DkoeSHE7yP5OcOdT2oySHutfoBeVZ13V5kpWh7X9oqO2yJH/ZvS4bfe+M6/rUUE1/keR7Q22zHK+bkjyV5IFV2pPk013dh5OcN9Q2y/GaVNe/6eq5P8k3k/zCUNtj3fJDSZbnXNcFSZ4b+nldM9S25j4w47r+41BND3T71Gu6tpmMV5IdSe7qcuDBJB8d02e2+1dVbZoXg4uu3wVeD7wMuA9440if3wKu76Z3A7d102/s+p8KnNWtZ8sc6/ol4Ke76Q+/WFc3f+wkjtflwHVj3vsa4NHu31d306+eV10j/f89g4vtMx2vbt3/HDgPeGCV9ouAO4AAbwUOznq8etb19he3x+BjOA4OtT0GbDtJ43UB8JWN7gPTrmuk768wuPNupuMFbAfO66ZfCfzFmP8fZ7p/bbYj9I18zMAu4HNV9UJV/RVwpFvfXOqqqruq6gfd7N0M7teftT7jtZpfBg5U1TNV9SxwALjwJNX1AeDWKW17TVX1dQZ3Yq1mF3BLDdwNnJZkO7Mdr4l1VdU3u+3C/PavPuO1mo3sm9Ouay77V1U9WVX3dtPfBx7mx5+qn+n+tdkCfSMfM9DnvbOsa9gVDH4Lv+jlSZaT3J3kX02ppvXU9b7uz7vbk7z4kNimGK/u1NRZwJ1Di2c1Xn2sVvssx2u9RvevAv57knsyeBp73t6W5L4kdyQ5p1u2KcYryU8zCMYvDC2e+XhlcCr4XODgSNNM96+5Pvr/kyDJrwFLwL8YWnxmVR1N8nrgziT3V9V351TSl4Fbq+qFJP+OwV8375zTtvvYDdxeVT8aWnYyx2tTS/JLDAL9F4cW/2I3Xj8LHEjyne4Idh7uZfDzOpbkIuBLwNlz2nYfvwL8r6oaPpqf6XgleQWDXyAfq6rnp7XePjbbEfp6PmaAvPRjBvq8d5Z1keRfAh8HLqmqF15cXlVHu38fBb7G4Df3XOqqqqeHarkBeEvf986yriG7GflzeIbj1cdqtc9yvHpJ8iYGP8NdVfUPH60xNF5PAf+N6Z1qnKiqnq+qY930fmBrkm1sgvHqrLV/TX28kmxlEOafraovjuky2/1r2hcGNnhR4RQGFwPO4h8vpJwz0ucjvPSi6Oe76XN46UXRR5neRdE+dZ3L4CLQ2SPLXw2c2k1vA/6SKV0c6lnX9qHpfw3cXf94Eeavuvpe3U2/Zl51df1+jsEFqsxjvIa2scjqF/ku5qUXrb416/HqWddOBteF3j6y/GeAVw5NfxO4cI51ve7Fnx+DYPzrbux67QOzqqtrfxWD8+w/M4/x6v67bwH+cI0+M92/pja4U/whXcTg6vB3gY93y65lcNQL8HLgv3Y797eA1w+99+Pd+x4B3jPnuv4H8DfAoe61r1v+duD+boe+H7hiznX9F+DBbvt3AT839N5/243jEeCD86yrm//PwCdG3jfr8boVeBL4PwzOU14B/Cbwm117GHyhy3e77S/Nabwm1XUD8OzQ/rXcLX99N1b3dT/nj8+5riuH9q+7GfqFM24fmFddXZ/LGdwoMfy+mY0Xg9NgBRwe+jldNM/9yydFJakRm+0cuiTpBBnoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ14v8BrzypWYN4D4MAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "# Plot true distribution\n",
        "plt.hist(y, density=True)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a4MJch1pgPOv"
      },
      "source": [
        "# Design of Single-Machine Simulation of Federated Learning System\n",
        "\n",
        "We use a controlled environment that is suitable for experiments. We assume\n",
        "a synchronous update scheme that proceeds in rounds of\n",
        "communication. There is a fixed set of $K$ clients, each\n",
        "with a fixed local dataset. At the beginning of each round, the server\n",
        "sends the current global model parameters to selected clients. Each selected client then trains its model on its local dataset, and sends an\n",
        "update to the server. The server aggregates these updates and applies them\n",
        "to the global model, and the cycle repeats [(McMahan et al., 2017)](https://arxiv.org/pdf/1602.05629.pdf). In our simulation, of cross-silo FL, all clients are selected for each round of federated learning."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OJxTmegnJZwv"
      },
      "source": [
        "## Federated Evaluation\n",
        "\n",
        "There are two main approaches to evaluate models in a federated learning system: centralized evaluation and federated evaluation. In centralized evaluation, the aggregated model is validated on the server-side, and the validation set is sampled from the population's probability distribution. In federated evaluation, the aggregated model is validated on the client-side, and the validation set is sampled from the same probability distribution as the client's training set [(Adap GmbH, 2022)](https://flower.dev/docs/evaluation.html).\n",
        "\n",
        "We focus on federated evaluation as it's typically used in real-world applications."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ASV3jBjpqf97"
      },
      "outputs": [],
      "source": [
        "# Initialize global experiment config\n",
        "DATA_PARTITIONING_STRATEGY = \"\"\n",
        "DATA_PARTITIONING_RANDOM_SEED = 0\n",
        "DIRICHLET_DISTRIBUTION_PARAM = 10\n",
        "NUMBER_OF_CLIENTS = 10\n",
        "MIN_AVAILABLE_CLIENTS = NUMBER_OF_CLIENTS\n",
        "NUMBER_OF_FL_ROUNDS = 50\n",
        "LOCAL_EPOCHS = ceil(1500 / NUMBER_OF_FL_ROUNDS)\n",
        "MIN_FIT_CLIENTS = NUMBER_OF_CLIENTS\n",
        "FRACTION_FIT = 1.0\n",
        "FRACTION_FIT_RANDOM_SEED = 1\n",
        "MIN_EVAL_CLIENTS = NUMBER_OF_CLIENTS\n",
        "FRACTION_EVAL = 1.0\n",
        "EVAL_STRATEGY = \"federated\"\n",
        "DATA_SHARING = False\n",
        "NUMBER_OF_OBSERVATIONS_SELECTED_WITH_DATA_PARTITIONING_STRATEGY = 10\n",
        "NUMBER_OF_VALIDATION_FOLDS = 10\n",
        "NUMBER_OF_REPEATS = NUMBER_OF_OBSERVATIONS_SELECTED_WITH_DATA_PARTITIONING_STRATEGY * NUMBER_OF_VALIDATION_FOLDS\n",
        "LOG_DIR = \"\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-oaUWAyYN2hp"
      },
      "source": [
        "Key:\n",
        "\n",
        "DATA_PARTITIONING_STRATEGY - strategy for distributing samples across clients\n",
        "\n",
        "DATA_PARTITIONING_RANDOM_SEED - seed to initialize random data partition generator, set for experimental control and reproducability\n",
        "\n",
        "DIRICHLET_DISTRIBUTION_PARAM - distribution parameter of the Dirichlet distribution, used in particular data partitioning strategies to control degree of imbalance\n",
        "\n",
        "NUMBER_OF_CLIENTS - total number of clients participating in the FL\n",
        "\n",
        "MIN_AVAILABLE_CLIENTS - minimum number of clients that must be connected to the server before FL can begin\n",
        "\n",
        "NUMBER_OF_FL_ROUNDS - number of FL rounds, set to a sufficiently large number so that the performance of the FL system can be observed\n",
        "\n",
        "LOCAL_EPOCHS - number of complete passes over local training data performed by clients during a training round, limited by the maximum number of iterations taken for the solvers to converge and the number of FL rounds, so that the performance of the FL system can be observed\n",
        "\n",
        "MIN_FIT_CLIENTS - minimum number of clients used during training\n",
        "\n",
        "FRACTION_FIT - fraction of random clients selected to participate in a training round\n",
        "\n",
        "FRACTION_FIT_RANDOM_SEED - seed for initializing the random sampling used to select clients for participation in a training round, set for experimental control and reproducability\n",
        "\n",
        "MIN_EVAL_CLIENTS - minimum number of clients used during evaluation\n",
        "\n",
        "FRACTION_EVAL - fraction of random clients selected to participate in an evaluation round\n",
        "\n",
        "EVAL_STRATEGY - there are two main approaches to evaluate models in federated learning systems: centralized (server-side) evaluation and federated (client-side) evaluation\n",
        "\n",
        "DATA_SHARING - whether or not to use a data-sharing strategy [(Zhao et al., 2018)](https://arxiv.org/pdf/1806.00582.pdf) to improve training on non-IID data\n",
        "\n",
        "NUMBER_OF_OBSERVATIONS_SELECTED_WITH_DATA_PARTITIONING_STRATEGY - number of observations drawn using each data partitioning strategy\n",
        "\n",
        "NUMBER_OF_VALIDATION_FOLDS - number of folds used in cross-validation\n",
        "\n",
        "NUMBER_OF_REPEATS - total number of experiment repeats\n",
        "\n",
        "LOG_DIR - folder in which to log experiments"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ne1N9RZuiwiE"
      },
      "source": [
        "## Utility Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8KXr8FmTgNrM"
      },
      "outputs": [],
      "source": [
        "LogRegParams = Tuple[np.ndarray, np.ndarray]\n",
        "\n",
        "def get_model_parameters(model: LogisticRegression) -> LogRegParams:\n",
        "#Returns the paramters of a sklearn LogisticRegression model.\n",
        "    params = (model.coef_, model.intercept_)\n",
        "    return params\n",
        "\n",
        "\n",
        "def set_model_params(model: LogisticRegression, params: LogRegParams) -> LogisticRegression:\n",
        "#Sets the parameters of a sklean LogisticRegression model.\n",
        "    model.coef_ = params[0]\n",
        "    model.intercept_ = params[1]\n",
        "    return model\n",
        "\n",
        "\n",
        "def set_initial_params(model: LogisticRegression):\n",
        "#Sets initial parameters as zeros Required since model params are\n",
        "#uninitialized until model.fit is called.\n",
        "#But server asks for initial parameters from clients at launch. Refer\n",
        "#to sklearn.linear_model.LogisticRegression documentation for more\n",
        "#information.\n",
        "\n",
        "    if DATA_SHARING:\n",
        "      model.classes_ = warm_up_model.classes_\n",
        "      model.coef_ = warm_up_model.coef_\n",
        "      model.intercept_ = warm_up_model.intercept_\n",
        "    else:\n",
        "      n_classes = 3\n",
        "      n_features = X_.shape[1]  # Number of features in dataset after feature selection by transformer\n",
        "\n",
        "      model.classes_ = np.array([i for i in range(n_classes)])\n",
        "      model.coef_ = np.zeros((n_classes, n_features))\n",
        "      model.intercept_ = np.zeros((n_classes,))\n",
        "\n",
        "def get_partitions(x, y):\n",
        "\n",
        "    '''\n",
        "    Adapted from:\n",
        "\n",
        "    Title: NIID-Bench\n",
        "    Authors: Li et al.\n",
        "    Date: Nov 17, 2021\n",
        "    Availability: https://github.com/Xtra-Computing/NIID-Bench/blob/main/utils.py\n",
        "\n",
        "    '''\n",
        "\n",
        "    np.random.seed(DATA_PARTITIONING_RANDOM_SEED)\n",
        "\n",
        "    n_train = y.shape[0]\n",
        "\n",
        "    if DATA_PARTITIONING_STRATEGY == \"iid\":\n",
        "        idxs = np.random.permutation(n_train)\n",
        "        batch_idxs = np.array_split(idxs, NUMBER_OF_CLIENTS)\n",
        "        net_dataidx_map = {i: batch_idxs[i] for i in range(NUMBER_OF_CLIENTS)}\n",
        "    elif DATA_PARTITIONING_STRATEGY == \"noniid-distribution-based-label-imbalance\":\n",
        "        min_size = 0\n",
        "        min_require_size = 10\n",
        "        K = 3\n",
        "        N = y.shape[0]\n",
        "        net_dataidx_map = {}\n",
        "\n",
        "        while min_size < min_require_size:\n",
        "            idx_batch = [[] for _ in range(NUMBER_OF_CLIENTS)]\n",
        "\n",
        "            for k in range(K):\n",
        "                idx_k = np.where(y == k)[0]\n",
        "                np.random.shuffle(idx_k)\n",
        "                proportions = np.random.dirichlet(np.repeat(DIRICHLET_DISTRIBUTION_PARAM, NUMBER_OF_CLIENTS))\n",
        "                proportions = np.array([p * (len(idx_j) < N / NUMBER_OF_CLIENTS) for p, idx_j in zip(proportions, idx_batch)])\n",
        "                proportions = proportions / proportions.sum()\n",
        "                proportions = (np.cumsum(proportions) * len(idx_k)).astype(int)[:-1]\n",
        "                idx_batch = [idx_j + idx.tolist() for idx_j, idx in zip(idx_batch, np.split(idx_k, proportions))]\n",
        "                min_size = min([len(idx_j) for idx_j in idx_batch])\n",
        "\n",
        "        for j in range(NUMBER_OF_CLIENTS):\n",
        "            np.random.shuffle(idx_batch[j])\n",
        "            net_dataidx_map[j] = idx_batch[j]\n",
        "    elif DATA_PARTITIONING_STRATEGY == \"noniid-quantity-based-label-imbalance-1\" or DATA_PARTITIONING_STRATEGY == \"noniid-quantity-based-label-imbalance-2\":\n",
        "        num = int(DATA_PARTITIONING_STRATEGY[38]) # number of distinct labels per client\n",
        "        K = 3 # total number of labels\n",
        "        times = [0 for _ in range(K)] # holds the number of parties to which each label ID is assigned\n",
        "        contain = [[] for _ in range(NUMBER_OF_CLIENTS)] # holds the buckets of num label IDs assigned to each party\n",
        "\n",
        "        for i in range(K): # ensure each label ID is assigned\n",
        "            bucket = contain[np.random.randint(NUMBER_OF_CLIENTS)]\n",
        "            while len(bucket) == num:\n",
        "                bucket = contain[np.random.randint(NUMBER_OF_CLIENTS)]\n",
        "            bucket.append(i)\n",
        "            times[i] += 1\n",
        "\n",
        "        for bucket in contain: # randomly assign num different distinct label IDs to each party\n",
        "            j = len(bucket)\n",
        "            while j < num:\n",
        "                ind = np.random.randint(K)\n",
        "                if ind not in bucket:\n",
        "                    bucket.append(ind)\n",
        "                    times[ind] += 1\n",
        "                    j += 1\n",
        "\n",
        "        net_dataidx_map ={i: np.ndarray(0, dtype=np.int64) for i in range(NUMBER_OF_CLIENTS)}\n",
        "\n",
        "        # For the samples of each label, randomly and equally divide them into the parties\n",
        "        # which own the label, such that there is no overlap between the samples of different parties\n",
        "        for i in range(K):\n",
        "            idx_k = np.where(y == i)[0]\n",
        "            np.random.shuffle(idx_k)\n",
        "            split = np.array_split(idx_k,times[i])\n",
        "            ids = 0\n",
        "            for j in range(NUMBER_OF_CLIENTS):\n",
        "                if i in contain[j]:\n",
        "                    net_dataidx_map[j]=np.append(net_dataidx_map[j],split[ids])\n",
        "                    ids += 1\n",
        "\n",
        "    elif DATA_PARTITIONING_STRATEGY == \"iid-quantity-skew\":\n",
        "        idxs = np.random.permutation(n_train)\n",
        "        min_size = 0\n",
        "        while min_size < 10:\n",
        "            proportions = np.random.dirichlet(np.repeat(DIRICHLET_DISTRIBUTION_PARAM, NUMBER_OF_CLIENTS))\n",
        "            proportions = proportions/proportions.sum()\n",
        "            min_size = np.min(proportions*len(idxs))\n",
        "        proportions = (np.cumsum(proportions)*len(idxs)).astype(int)[:-1]\n",
        "        batch_idxs = np.split(idxs, proportions)\n",
        "        net_dataidx_map = {i: batch_idxs[i] for i in range(NUMBER_OF_CLIENTS)}\n",
        "\n",
        "    partitioned_train_set = [(x[net_dataidx_map[idx]], pd.Series(y[net_dataidx_map[idx]])) for idx in range(NUMBER_OF_CLIENTS)]\n",
        "\n",
        "    return partitioned_train_set\n",
        "\n",
        "\n",
        "def get_train_partitions(x_train, y_train):\n",
        "  partitioned_train_set = []\n",
        "  \n",
        "  if DATA_SHARING:\n",
        "    partitioned_train_set = get_partitions(X_train_client, y_train_client.to_numpy())\n",
        "    # a random α portion of G is merged with the data of each client\n",
        "    partitioned_train_set = [(np.concatenate((X_train_partition, X_train_shared)), pd.concat((y_train_partition, y_train_shared))) for X_train_partition, y_train_partition in partitioned_train_set]\n",
        "  else:\n",
        "    partitioned_train_set = get_partitions(x_train, y_train)\n",
        "\n",
        "  return partitioned_train_set\n",
        "\n",
        "\n",
        "def get_test_partitions(x_test, y_test, partitioned_train_set):\n",
        "  partitioned_test_set = []\n",
        "\n",
        "  if DATA_PARTITIONING_STRATEGY == \"noniid-distribution-based-label-imbalance\" or DATA_PARTITIONING_STRATEGY == \"iid-quantity-skew\":\n",
        "    # Grody estimate of distribution\n",
        "\n",
        "    np.random.seed(DATA_PARTITIONING_RANDOM_SEED)\n",
        "    \n",
        "    net_dataidx_map = {i: np.ndarray(0, dtype=np.int64) for i in range(NUMBER_OF_CLIENTS)}\n",
        "\n",
        "    total_label_freqs = 0\n",
        "\n",
        "    if DATA_SHARING:\n",
        "      total_label_freqs = y_train_client.value_counts()\n",
        "    else:\n",
        "      total_label_freqs = y_train.value_counts()\n",
        "\n",
        "    for i in range(3):\n",
        "      idx_i = np.where(y_test == i)[0]\n",
        "      np.random.shuffle(idx_i)\n",
        "\n",
        "      indices = []\n",
        "\n",
        "      offset = 0\n",
        "\n",
        "      for j in range(NUMBER_OF_CLIENTS - 1):\n",
        "        partition_label_proportion = partitioned_train_set[j][1].value_counts().reindex(range(3), fill_value=0)[i] / total_label_freqs[i]\n",
        "        partition_label_freq = int(len(idx_i )*partition_label_proportion)\n",
        "        split_index = offset + partition_label_freq\n",
        "        indices.append(split_index)\n",
        "        offset = split_index\n",
        "      \n",
        "      split = np.array_split(idx_i, indices)\n",
        "\n",
        "      for j in range(NUMBER_OF_CLIENTS):\n",
        "        net_dataidx_map[j]=np.append(net_dataidx_map[j],split[j])\n",
        "\n",
        "    partitioned_test_set = [(x_test[net_dataidx_map[idx]], pd.Series(y_test[net_dataidx_map[idx]])) for idx in range(NUMBER_OF_CLIENTS)]\n",
        "  else:\n",
        "    partitioned_test_set = get_partitions(x_test, y_test)\n",
        "\n",
        "  return partitioned_test_set"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ngfm4O3Bj_7-"
      },
      "source": [
        "## Server Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kUGPEvkTkL2M"
      },
      "outputs": [],
      "source": [
        "def fit_round(rnd: int) -> Dict:\n",
        "  \"\"\"Send fit round number to client.\"\"\"\n",
        "\n",
        "  return {\"rnd\": rnd}\n",
        "\n",
        "def evaluate_round(rnd: int) -> Dict:\n",
        "  \"\"\"Send evaluate round number to client.\"\"\"\n",
        "\n",
        "  return {\"rnd\": rnd}\n",
        "\n",
        "# get_eval_fn is a closure factory function that creates a closure consisting of the inner function evaluate together with its enclosing environment.\n",
        "# The closure can modify the mutable objects aka dynamic enclosing state, such as model, within its enclosing state\n",
        "def get_eval_fn(model: LogisticRegression, y_preds: ListProxy, test_accs: ListProxy):\n",
        "    \"\"\"Return an evaluation function for server-side evaluation.\"\"\"\n",
        "\n",
        "    # The `evaluate` function will be called after every training round.\n",
        "    def evaluate(parameters: fl.common.Weights):\n",
        "        # print(\"Beginning centralized evaluation...\")\n",
        "        # print(\"Updating global model with the latest parameters...\")\n",
        "        # Update model with the latest parameters\n",
        "        set_model_params(model, parameters)\n",
        "        # print(\"Predicting class labels for samples in X_test...\")\n",
        "        # Predict class labels for samples in X_test\n",
        "        predictions = model.predict(X_test)\n",
        "        y_preds.append(predictions)\n",
        "        # Calculate probability estimates\n",
        "        # print(\"Calculating probability estimates...\")\n",
        "        prob_estimates = model.predict_proba(X_test)\n",
        "        # Calculate cross-entropy loss\n",
        "        # print(\"Calculating cross-entropy loss...\")\n",
        "        loss = log_loss(y_test, prob_estimates)\n",
        "        # print(\"Calculating the mean accuracy on the test data and labels...\")\n",
        "        # Calculate the mean accuracy on the test data and labels.\n",
        "        accuracy = model.score(X_test, y_test)\n",
        "        test_accs.append(accuracy)\n",
        "        # print(\"Finished centralized evaluation.\")\n",
        "        return loss, {\"accuracy\": accuracy}\n",
        "        \n",
        "    return evaluate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0Y_R6Wa9wJxh"
      },
      "outputs": [],
      "source": [
        "def my_aggregate(results: List[Tuple[Weights, int]]) -> Weights:\n",
        "    \"\"\"Compute weighted average.\"\"\"\n",
        "    # Calculate the total number of examples used during training\n",
        "    num_examples_total = sum([num_examples for _, num_examples in results])\n",
        "\n",
        "    # print(\"Parameters received from each client:\")\n",
        "    # for parameters, num_examples in results:\n",
        "      # print(f\"Num_examples: {num_examples}\")\n",
        "      # print(f\"Shape of coeffs: {parameters[0].shape}\")  \n",
        "      # print(f\"Coeffs: {parameters[0]}\")\n",
        "      # print(f\"Shape of biases: {parameters[1].shape}\")  \n",
        "      # print(f\"Biases: {parameters[1]}\") \n",
        "\n",
        "    # Create a list of weights, each multiplied by the related number of examples\n",
        "    weighted_parameters = [\n",
        "        [parameter * num_examples for parameter in parameters] for parameters, num_examples in results\n",
        "    ]\n",
        "\n",
        "    # print(f\"Weighted parameters: {weighted_parameters}\")\n",
        "\n",
        "    # print(f\"Zipped weighted parameters: {list(zip(*weighted_parameters))}\")\n",
        "\n",
        "    # Compute average parameters\n",
        "    weights_prime = [\n",
        "        reduce(np.add, layer_updates) / num_examples_total\n",
        "        for layer_updates in zip(*weighted_parameters)\n",
        "    ]\n",
        "\n",
        "    return weights_prime"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yMirYHyIbfo5"
      },
      "outputs": [],
      "source": [
        "def start_flower_server(y_preds: ListProxy, test_accs: ListProxy):\n",
        "  \n",
        "  if EVAL_STRATEGY == \"centralized\":\n",
        "    model = LogisticRegression()\n",
        "    set_initial_params(model)\n",
        "\n",
        "# Sample fraction of available clients for the next round\n",
        "# Minimum number of clients to be sampled for the next round\n",
        "# Minimum number of clients that need to be connected to the server before a round of federated learning can start. If few clients \n",
        "# are connected then the server will wait until more connect before sampling clients\n",
        "# There are two main approaches to evaluate models in flower, centralized (server-side) evaluation and federated (client-side) evaluation.\n",
        "# Centralized and federated evaluation are mutually exclusive.\n",
        "# All built-in strategies support centalized evaluation by providing an evaluation function during \n",
        "# initialization. An evaluation function is any function that can take the current global model parameters as input \n",
        "# and return evaluation results. The current server implementation calls eval_fn AFTER parameter aggregation  \n",
        "# (training round).\n",
        "# Customizing the existing FedAvg strategy with the callback function fit_round, bound to eval_fn, to pass variable \n",
        "# configuration values to the client each training round. The provided function will be called by the strategy \n",
        "# and must return a dictionary of arbitrary configuration key-value pairs that will be sent to the client \n",
        "# (received in its own client.fit function).\n",
        "\n",
        "  class MyClientManager(fl.server.SimpleClientManager):\n",
        "    def __init__(self) -> None:\n",
        "      super().__init__()\n",
        "      random.seed(FRACTION_FIT_RANDOM_SEED)\n",
        "\n",
        "  client_manager = MyClientManager()\n",
        "\n",
        "  class MyStrategy(fl.server.strategy.FedAvg):\n",
        "    def aggregate_fit(self, rnd: int, results: List[Tuple[ClientProxy, FitRes]], failures: List[BaseException]) -> Tuple[Optional[Parameters], Dict[str, Scalar]]:\n",
        "        \"\"\"Aggregate fit results using weighted average.\"\"\"\n",
        "        if not results:\n",
        "            return None, {}\n",
        "\n",
        "        # Do not aggregate if there are failures and failures are not accepted\n",
        "        if not self.accept_failures and failures:\n",
        "            return None, {}\n",
        "\n",
        "        for i in range(len(results)):\n",
        "          client, fit_res = results[i]\n",
        "\n",
        "        # Convert results\n",
        "        weights_results = [(parameters_to_weights(fit_res.parameters), fit_res.num_examples)\n",
        "            for client, fit_res in results\n",
        "        ]\n",
        "\n",
        "        average_weights = my_aggregate(weights_results)\n",
        "\n",
        "        # print(f\"Average parameters after aggregation: {average_weights}, number of coeffs: {average_weights[0].shape}, number of biases: {average_weights[1].shape}\")\n",
        "        \n",
        "        return weights_to_parameters(average_weights), {}\n",
        "\n",
        "  strategy = MyStrategy()     \n",
        "\n",
        "  # strategy = fl.server.strategy.FedAvg()\n",
        "\n",
        "  strategy.min_available_clients = MIN_AVAILABLE_CLIENTS\n",
        "  strategy.fraction_fit = FRACTION_FIT\n",
        "  strategy.min_fit_clients = MIN_FIT_CLIENTS          \n",
        "  strategy.on_fit_config_fn = fit_round\n",
        "  strategy.fraction_eval = FRACTION_EVAL\n",
        "  strategy.min_eval_clients = MIN_EVAL_CLIENTS\n",
        "\n",
        "  if (EVAL_STRATEGY == \"federated\"):\n",
        "    strategy.on_evaluate_config_fn = evaluate_round\n",
        "  else:\n",
        "    strategy.eval_fn = get_eval_fn(model, y_preds, test_accs)\n",
        "  \n",
        "  server = fl.server.Server(client_manager=client_manager, strategy=strategy)\n",
        "  \n",
        "  fl.server.start_server(\"0.0.0.0:80\", server=server, config={\"num_rounds\": NUMBER_OF_FL_ROUNDS})"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "txBxYsxvsQ_2"
      },
      "source": [
        "## Client Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pi5hoX8kwx5q"
      },
      "outputs": [],
      "source": [
        "def start_flower_client(X_train: np.ndarray, y_train: np.ndarray, X_test: np.ndarray, y_test: np.ndarray, id: str, y_preds: ListProxy, test_accs: ListProxy):\n",
        "  pid = os.getpid()\n",
        "  print(f\"initializing Client{id} with pid {pid}\")\n",
        "\n",
        "  # Create LogisticRegression Model\n",
        "  # max_iter: way to limit the number of epochs. One epoch means that each sample in the training dataset has had an opportunity \n",
        "  # to update the internal model parameters. The solver iterates until convergence (determined by ‘tol’) or this number of iterations.\n",
        "  # i.e., the learning will be stopped either when the solver iterates max_iter times or when the cost between two iterations changes less than tol, \n",
        "  # whichever comes first. From sklearn.linear_model.LogisticRegression documentation we can see that ‘tol’ is 0.0001 by default.\n",
        "  # warm_start: prevent refreshing weights when fitting, this is required for federated learning\n",
        "  # As a result of the fact that warm_start is required, the variable solver must be changed because liblinear does not support warm start\n",
        "  # Framework used for multi class classification is multinomial. \n",
        "  # This is necessary because of the non-iid data in federated learning scenarios; in a multinomial framework, learning 𝛽𝑖 for a particular class i\n",
        "  # immediately affects our model for other classes j, through the joint model. Consequently, The training algorithm uses log loss aka cross-entropy loss\n",
        "  model = LogisticRegression(penalty='l2', max_iter=LOCAL_EPOCHS, warm_start=True, class_weight='balanced', C=0.01)\n",
        "\n",
        "  # Setting initial model parameters\n",
        "  set_initial_params(model)\n",
        "\n",
        "  # Define Flower client\n",
        "  class MyClient(fl.client.NumPyClient):\n",
        "    def get_parameters(self): # -> List[numpy.ndarray]\n",
        "      return get_model_parameters(model)\n",
        "\n",
        "    # Training happens in fit method and can be configured from the server side. \n",
        "    # Model parameters can also be evaluated during training. Client.fit can return arbitrary evaluation results as a dictionary\n",
        "    def fit(self, parameters, config):\n",
        "      # print(f\"Client{id} with pid {pid} is sampled in training rnd {config['rnd']}\")\n",
        "      set_model_params(model, parameters)\n",
        "      # Ignore convergence failure due to low local epochs\n",
        "      with warnings.catch_warnings():\n",
        "          warnings.simplefilter(\"ignore\")\n",
        "          model.fit(X_train, y_train)\n",
        "      # print(f\"Training finished for round {config['rnd']}\")\n",
        "      # print(f\"Model params is a 2 tuple, first element contains coefficients/weights, second element contains intercept\")\n",
        "      # print(f\"Number of coefficients/weights: {get_model_parameters(model)[0].shape}\")\n",
        "      # print(f\"Model params: {get_model_parameters(model)}\")\n",
        "      return get_model_parameters(model), len(X_train), {}\n",
        "\n",
        "    # Federated evaluation happens in the evaluate method and can be configured from the server side.\n",
        "    def evaluate(self, parameters, config): \n",
        "      # print(f\"Client{id} with pid {pid} is sampled in evaluation rnd {config['rnd']}\")\n",
        "      \n",
        "      # Update model with the latest parameters\n",
        "      set_model_params(model, parameters)\n",
        "\n",
        "      # Remind model of classes in the population\n",
        "      model.classes_ = np.array([0, 1, 2])\n",
        "      \n",
        "      # Predict class labels for samples in X_test\n",
        "      predictions = model.predict(X_test)\n",
        "\n",
        "      y_preds.append(predictions)\n",
        "      \n",
        "      # Calculate probability estimates\n",
        "      prob_estimates = model.predict_proba(X_test)\n",
        "\n",
        "      # Calculate cross-entropy loss\n",
        "      loss = log_loss(y_test, prob_estimates, labels=[0,1,2])\n",
        "\n",
        "      # Calculate the mean accuracy on the test data and labels.\n",
        "      accuracy = model.score(X_test, y_test)\n",
        "      test_accs.append(accuracy)\n",
        "\n",
        "      return loss, len(X_test), {\"accuracy\": accuracy}\n",
        "\n",
        "  client = MyClient()\n",
        "\n",
        "  # Start Flower client\n",
        "  fl.client.start_numpy_client(\"0.0.0.0:80\", client=client)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7ulkjIuLfdJg"
      },
      "source": [
        "# Test Bench"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FKQ2RLAiS_Y3"
      },
      "source": [
        "## Test Bench Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rHtmGyqNVyQj"
      },
      "outputs": [],
      "source": [
        "# Define function for getting a time stamp\n",
        "def get_time_stamp():\n",
        "  return datetime.today().strftime('%d_%m_%Y__%H_%M_%S')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qd1kcs6Ir_Ur"
      },
      "outputs": [],
      "source": [
        "def evaluate_local_models(partitioned_train_set, partitioned_test_set):\n",
        "  local_models_y_preds = []\n",
        "\n",
        "  for i in range(NUMBER_OF_CLIENTS):\n",
        "    (X_train_partition, y_train_partition) = partitioned_train_set[i]\n",
        "    model = classifier.fit(X_train_partition, y_train_partition)\n",
        "\n",
        "    (X_test_partition, y_test_partition) = partitioned_test_set[i]\n",
        "    y_preds = model.predict(X_test_partition)\n",
        "    local_models_y_preds.append(y_preds)\n",
        "\n",
        "  return local_models_y_preds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7sl2VPkKgdTX"
      },
      "outputs": [],
      "source": [
        "# Define function for running simulation with global config\n",
        "def run_sim(partitioned_train_set, partitioned_test_set=None):\n",
        "  with Manager() as manager:\n",
        "    # Holds all child processes\n",
        "    processes = []\n",
        "\n",
        "    # List proxy for gathering class labels predicted during centralized evaluation\n",
        "    centralized_y_preds = manager.list()\n",
        "\n",
        "    # List proxy for gathering test accuracies from centralized evaluation\n",
        "    centralized_test_accs = manager.list()\n",
        "\n",
        "    # Holds all the list proxies for gathering class labels predicted during federated evaluation\n",
        "    federated_y_preds = []\n",
        "\n",
        "    # Holds all the list proxies for gathering test accuracies from federated evaluation\n",
        "    federated_test_accs = []\n",
        "\n",
        "    # Start Flower server\n",
        "    server_process = Process(target=start_flower_server, args=(centralized_y_preds, centralized_test_accs))\n",
        "    server_process.start()\n",
        "    processes.append(server_process)\n",
        "\n",
        "    # Block the script here for a second or two so the server has time to start\n",
        "    time.sleep(2)\n",
        "\n",
        "    # Start Flower clients\n",
        "    if EVAL_STRATEGY == \"centralized\":\n",
        "      for i in range(NUMBER_OF_CLIENTS):\n",
        "        y_preds = manager.list()\n",
        "        test_accs = manager.list()\n",
        "        (X_train_partition, y_train_partition) = partitioned_train_set[i]\n",
        "        client_process = Process(target=start_flower_client, args=(X_train_partition, y_train_partition, X_test, y_test, str(i), y_preds, test_accs))\n",
        "        client_process.start()\n",
        "\n",
        "        federated_y_preds.append(y_preds)\n",
        "        federated_test_accs.append(test_accs)\n",
        "        processes.append(client_process)\n",
        "\n",
        "        # Block the script here for a second to ensure clients register with server in-order (helps control fraction fit variable)\n",
        "        time.sleep(1)\n",
        "    else:\n",
        "      for i in range(NUMBER_OF_CLIENTS):\n",
        "        y_preds = manager.list()\n",
        "        test_accs = manager.list()\n",
        "        (X_train_partition, y_train_partition) = partitioned_train_set[i]\n",
        "        \n",
        "        (X_test_partition, y_test_partition) = partitioned_test_set[i]\n",
        "\n",
        "        client_process = Process(target=start_flower_client, args=(X_train_partition, y_train_partition, X_test_partition, y_test_partition, str(i), y_preds, test_accs))\n",
        "        client_process.start()\n",
        "\n",
        "        federated_y_preds.append(y_preds)\n",
        "        federated_test_accs.append(test_accs)\n",
        "        processes.append(client_process)\n",
        "\n",
        "        # Block the script here for a second to ensure clients register with server in-order (helps control fraction fit variable)\n",
        "        time.sleep(1)\n",
        "\n",
        "    # Wait for processes to finish\n",
        "    for p in processes:\n",
        "      p.join()\n",
        "\n",
        "    # Dereference proxies and return results\n",
        "    return {\"centralized_y_preds\": centralized_y_preds[:], \"centralized_test_accs\": centralized_test_accs[:], \n",
        "            \"federated_y_preds\": [y_preds[:] for y_preds in federated_y_preds], \"federated_test_accs\": [test_accs[:] for test_accs in federated_test_accs]}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YMzhuFFsb3nj"
      },
      "outputs": [],
      "source": [
        "def get_label_freqs(partitioned_train_set, relative=False):\n",
        "  label_freqs = [y_train.value_counts(normalize=relative).rename(\"y_train\")]\n",
        "  \n",
        "  for i in range(NUMBER_OF_CLIENTS):\n",
        "    label_freqs.append(partitioned_train_set[i][1].value_counts(normalize=relative).rename(\"client\" + str(i)))\n",
        "\n",
        "  label_freqs_table = pd.concat(label_freqs, axis=1).fillna(0)\n",
        "\n",
        "  if relative:\n",
        "    return label_freqs_table.astype(\"float64\")\n",
        "  else:\n",
        "    return label_freqs_table.astype(\"int64\")\n",
        "\n",
        "\n",
        "def plot_distribution_of_labels(partitioned_train_set, filepath=None):\n",
        "  # Resize figure\n",
        "  plt.figure(figsize=(rcParams[\"figure.figsize\"][0]*1.4, rcParams[\"figure.figsize\"][1]))\n",
        "\n",
        "  number_of_label_across_partitions = pd.concat([partitioned_train_set[i][1].value_counts().sort_index() for i in range(NUMBER_OF_CLIENTS)], axis=1).fillna(0).astype(\"int64\").to_numpy()\n",
        "\n",
        "  max_number_of_labels = max([len(partitioned_train_set[i][1]) for i in range(NUMBER_OF_CLIENTS)])\n",
        "  \n",
        "  labs = [str(i) for i in range(NUMBER_OF_CLIENTS)]\n",
        "\n",
        "  plt.barh(labs, number_of_label_across_partitions[0])\n",
        "  plt.barh(labs, number_of_label_across_partitions[1], left=number_of_label_across_partitions[0], color='g')\n",
        "  plt.barh(labs, number_of_label_across_partitions[2], left=number_of_label_across_partitions[0]+number_of_label_across_partitions[1], color='r')\n",
        "\n",
        "  plt.title(\"Distribution of Labels\")\n",
        "  plt.xlabel(\"Number of Local Data Samples\")\n",
        "  plt.ylabel(\"Client ID\")\n",
        "\n",
        "  labelIDs=['hate speech','offensive','neither']\n",
        "\n",
        "  plt.legend(labelIDs)\n",
        "\n",
        "  plt.xlim([0, max_number_of_labels + 300])\n",
        "  \n",
        "  if filepath:\n",
        "    plt.ioff()\n",
        "    plt.savefig(filepath, bbox_inches='tight')\n",
        "    plt.close()\n",
        "  else:\n",
        "    plt.show()\n",
        "\n",
        "\n",
        "def plot_performance_of_local_models(precision_recall_fscores_of_local_models, precision_recall_fscores_of_global_model, classification_metric, filepath=None):\n",
        "  classification_metrics = {'Precision': 0, 'Recall': 1, 'F1-score': 2}\n",
        "\n",
        "  labs = [str(i) for i in range(NUMBER_OF_CLIENTS)]\n",
        "\n",
        "  scaling_parameter = 5.5\n",
        "  x = np.arange(0, scaling_parameter*len(labs), scaling_parameter)  # the label locations\n",
        "  width = 2.2  # the width of the bars\n",
        "\n",
        "  fig, ax = plt.subplots(figsize=(rcParams[\"figure.figsize\"][0]*1.8, rcParams[\"figure.figsize\"][1] * 1.5))\n",
        "  rects1 = ax.bar(x - width/2, precision_recall_fscores_of_local_models[classification_metrics[classification_metric]], width, label='Local Model')\n",
        "  rects2 = ax.bar(x + width/2, precision_recall_fscores_of_global_model[classification_metrics[classification_metric]], width, label='Global Model')\n",
        "\n",
        "  # Add some text for labels, title and custom x-axis tick labels, etc.\n",
        "  ax.set_ylabel(classification_metric)\n",
        "  ax.set_xlabel('Client ID')\n",
        "  ax.set_title(classification_metric + ' of Local Models Against That of Global Model')\n",
        "  ax.set_xticks(x)\n",
        "  ax.set_xticklabels(labs)\n",
        "  ax.legend()\n",
        "\n",
        "  def autolabel(rects):\n",
        "    '''\n",
        "    Copied from:\n",
        "\n",
        "    Title: Matplotlib\n",
        "    Authors: John Hunter et al.\n",
        "    Date: 2012\n",
        "    Availability: https://matplotlib.org/3.2.2/gallery/lines_bars_and_markers/barchart.html\n",
        "\n",
        "    '''\n",
        "\n",
        "    \"\"\"Attach a text label above each bar in *rects*, displaying its height.\"\"\"\n",
        "    for rect in rects:\n",
        "      height = round(rect.get_height(), 2)\n",
        "      ax.annotate('{}'.format(height),\n",
        "                  xy=(rect.get_x() + rect.get_width() / 2, height),\n",
        "                  xytext=(0, 3),  # 3 points vertical offset\n",
        "                  textcoords=\"offset points\",\n",
        "                  ha='center', va='bottom')\n",
        "\n",
        "  autolabel(rects1)\n",
        "  autolabel(rects2)\n",
        "\n",
        "  plt.ylim([0, 1])\n",
        "  \n",
        "  if filepath:\n",
        "    plt.ioff()\n",
        "    plt.savefig(filepath)\n",
        "    plt.close()\n",
        "  else:\n",
        "    plt.show()\n",
        "\n",
        "\n",
        "def get_measures_of_dispersion(precision_recall_fscores):\n",
        "  return [np.std(classification_metric) for classification_metric in precision_recall_fscores]\n",
        "\n",
        "\n",
        "def precision_recall_fscore(y_true, y_pred):\n",
        "  return precision_recall_fscore_support(y_true, y_pred, average=\"macro\", zero_division=1)[:-1]\n",
        "\n",
        "\n",
        "def get_precision_recall_fscores_of_local_models(local_models_y_test, local_models_y_preds):\n",
        "  return list(zip(*[precision_recall_fscore(local_model_y_test, local_model_y_preds) for local_model_y_test, local_model_y_preds in zip(local_models_y_test, local_models_y_preds)]))\n",
        "\n",
        "\n",
        "def get_mean_precision_recall_fscores_of_local_models(precision_recall_fscores_of_local_models):\n",
        "  return [mean(metric) for metric in precision_recall_fscores_of_local_models]\n",
        "\n",
        "\n",
        "def get_utility_measure_on_distributions(aggr_model_precision_recall_fscore, avg_local_model_precision_recall_fscore):\n",
        "  arrays = [[\"Precision\", \"Precision\", \"Recall\", \"Recall\", \"F1-score\", \"F1-score\"],\n",
        "          [\"Aggr.\", \"Local\", \"Aggr.\", \"Local\", \"Aggr.\", \"Local\"]]\n",
        "  tuples = list(zip(*arrays))\n",
        "  index = pd.MultiIndex.from_tuples(tuples, names=[\"Distribution\", \"\"])\n",
        "\n",
        "  utility_measure = pd.DataFrame([[round(i, 2) for sub in zip(aggr_model_precision_recall_fscore, avg_local_model_precision_recall_fscore) for i in sub]], index=[DATA_PARTITIONING_STRATEGY], columns=index)\n",
        "  return utility_measure\n",
        "\n",
        "\n",
        "def plot_convergence_behaviour_from_federated_evaluation(federated_test_accs, filepath=None):\n",
        "  # Place legend outside axes\n",
        "  axes_legend_bbox = (1.03, 0.6)\n",
        "\n",
        "  # Resize figure\n",
        "  plt.figure(figsize=(rcParams[\"figure.figsize\"][0] + axes_legend_bbox[0] - 1, rcParams[\"figure.figsize\"][1]))\n",
        "\n",
        "  plt.plot([mean(test_accs) for test_accs in zip(*federated_test_accs)])\n",
        "  plt.ylim([0, 1])\n",
        "  plt.xlabel(\"Round\")\n",
        "  plt.ylabel(\"Avg Test Acc\")\n",
        "  plt.title(\"Convergence Behaviour\")\n",
        "  plt.legend(bbox_to_anchor = axes_legend_bbox)\n",
        "  \n",
        "  if filepath:\n",
        "    plt.ioff()\n",
        "    plt.savefig(filepath, bbox_inches='tight')\n",
        "    plt.close()\n",
        "  else:\n",
        "    plt.show()\n",
        "\n",
        "\n",
        "def plot_convergence_behaviour_from_centralized_evaluation(centralized_test_accs, filepath=None):\n",
        "  # Place legend outside axes\n",
        "  axes_legend_bbox = (1.26, 0.6)\n",
        "\n",
        "  # Resize figure\n",
        "  plt.figure(figsize=(rcParams[\"figure.figsize\"][0] + axes_legend_bbox[0] - 1, rcParams[\"figure.figsize\"][1]))\n",
        "\n",
        "  plt.plot(centralized_test_accs, label= 'FedShare' if DATA_SHARING else 'FedAvg')\n",
        "  plt.ylim([0, 1])\n",
        "  plt.xlabel(\"Round\")\n",
        "  plt.ylabel(\"Test Acc\")\n",
        "  plt.title(\"Convergence Behaviour\")\n",
        "  plt.legend(bbox_to_anchor = axes_legend_bbox)\n",
        "  \n",
        "  if filepath:\n",
        "    plt.ioff()\n",
        "    plt.savefig(filepath, bbox_inches='tight')\n",
        "    plt.close()\n",
        "  else:\n",
        "    plt.show()\n",
        "\n",
        "\n",
        "def report_classification_macro_averages(y_test, y_preds):\n",
        "  report = classification_report(y_test, y_preds)\n",
        "  rows = report.split(\"\\n\")\n",
        "  headings = \" \".join(rows[0].split(\" \")[:-1])\n",
        "  macro_averages = \" \".join(rows[7].split(\" \")[:-1])\n",
        "  return \"\\n\".join((headings, macro_averages))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0v3n5u-dWYiA"
      },
      "source": [
        "## Design of Experimental Validation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "01tB5t4Y-9J1"
      },
      "source": [
        "### Distributed Optimization Problem\n",
        "\n",
        "Our federated learning system has the finite sum objective with generic form:\n",
        "\n",
        "\\begin{equation}\n",
        "\\min_{w \\in \\mathbb{R}^d} f(w)\n",
        "\\qquad \\text{where} \\qquad\n",
        "f(w) \\overset{\\text{def}}{=} \\frac{1}{n} \\sum_{i=1}^n f_i(w).\n",
        "\\end{equation}\n",
        "\n",
        "where $n$ is the number of data samples and $w$ is the model parameters\n",
        "\n",
        "We take $f_i(w) = \\ell(x_i, y_i; w)$, that is, the log loss of the prediction on sample $(x_i, y_i)$ made with coefficients $w$. We assume there are $K$ clients over which the data is partitioned, with $\\mathcal{P}_k$ the set of samples on client $k$, with $n_k = |\\mathcal{P}_k|$. Thus, we can re-write the system's objective as:\n",
        "\n",
        "\\begin{equation}\n",
        "\\min_{w \\in \\mathbb{R}^d} \\sum_{k = 1}^K \\frac{n_k}{n} F_k(w)\n",
        "\\qquad \\text{where} \\qquad\n",
        "F_k(w) \\overset{\\text{def}}{=} \\frac{1}{n_k} \\sum_{i \\in \\mathcal{P}_k} f_i(w).\n",
        "\\end{equation}\n",
        "\n",
        "If the partition $\\mathcal{P}_k$ is formed by distributing the samples over the clients uniformly at random, then the f**IID assumption** made by distributed optimization algorithms holds[(Zhao et al., 2018)](https://arxiv.org/pdf/1806.00582.pdf). \n",
        "\n",
        "We use the distribution setting where the IID assumption holds as a basline for our experiments."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AzcJWzFYwoxZ"
      },
      "source": [
        "### Federated Optimization Problem\n",
        "\n",
        "The optimization problem implicit in federated learning is known as the federated optimization problem, drawing a contrast to the distributed optimization problem. The key properties that differentiate federated optimization are: unbalanced data (different clients can hold different amounts of data) and that the IID assumption does not hold.\n",
        "\n",
        "Here we focus on horizontal federated learning, where the clients share the same feature space but different sample space (in contrast to vertical federated learning, where the clients have different feature and sample spaces).\n",
        "\n",
        "An existing study gives a taxonomy of non-IID data distribution cases [(Kairouz et al., 2021)](https://arxiv.org/pdf/1912.04977.pdf). Consider the local data distribution $P(x_i, y_i)$. We can rewrite $P(x_i, y_i)$ as $P(x_i | y_i)P(y_i)$ and $P(y_i | x_i)P(x_i)$ so that we can more precisely show the differences between cases. The study gives 5 different non-IID cases:\n",
        "\n",
        "(1) label distribution skew (i.e., $P(y_i)$ is different among clients)\n",
        "\n",
        "(2) feature distribution skew (i.e., $P(x_i)$ is different among clients) \n",
        "\n",
        "(3) same labels but different features (i.e., $P(x_i|y_i)$ is different among clients) \n",
        "\n",
        "(4) same features but different labels (i.e., $P(y_i|x_i)$ is different among\n",
        "clients)\n",
        "\n",
        "(5) quantity skew (i.e., $P(x_i, y_i)$ is the same but the amount of data is different among clients) \n",
        "\n",
        "Here, (3) is mainly related to vertical FL (the clients share the same sample IDs but different features). (4) is not applicable in most FL studies, which assume same ground truths . (2) can be simulated on a dataset by adding different levels of noise to different partitions. In computer vision, these transformations can be done on the fly, using data generators e.g. for Gaussian noise. However, this isn't the case with NLP due to the grammatical structure of the text; a new carefully augmented dataset must be generated beforehand [(Shahul ES, 2021)](https://neptune.ai/blog/data-augmentation-nlp).\n",
        "\n",
        "Thus, we consider (5) quantity skew and (1) label distribution skew as possible non-IID data distribution cases in our experiments."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wjAQD_FZnaV-"
      },
      "source": [
        "### Unbalancedness (Quantity skew)\n",
        "\n",
        "The size of the local dataset, $|{\\mathcal{P}}_k|$, varies across clients. Although, the iid assumption still holds. We use a Dirichlet distribution to allocate different amounts of data samples to each client. We sample $q ∼ Dir_N (β)$, and allocate a $q_j$ proportion of the total data samples to ${\\mathcal{P}}_j$ . The distribution parameter, $β>0$, can be used to control the degree of imbalance. The smaller $β$ is, the more imbalanced the distribution [(Li et al., 2021)](https://arxiv.org/pdf/2102.02079.pdf).\n",
        "\n",
        "This scenario is relevant as irl some organisations will generate more data than others, leading to varying amounts of local training data."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-INaYNo9kF3v"
      },
      "source": [
        "### Label Distribution Skew: Distribution-based label imbalance\n",
        "\n",
        "Each client is allocated a proportion of the samples of each label according to a Dirichlet distribution. Dirichlet distribution is an appropriate\n",
        "choice to simulate imbalanced data distributions in the real-world [(Li et al., 2021)](https://arxiv.org/pdf/2102.02079.pdf). Specifically, we sample $p_k ∼ Dir_N(β)$ and allocate a $p_{k,j}$ proportion of the instances of class $k$ to client $j$. We fix to constrain the scope of the study. This partitioning strategy has been used to simulate real world data distributions in many recent FL studies [(Li et al., 2021)](https://arxiv.org/pdf/2102.02079.pdf)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NiPUE9JaktFE"
      },
      "source": [
        "### Label Distribution Skew: Quantity-based label imbalance\n",
        "\n",
        "Each client owns data samples of a fixed subset of total labels. Such a setting is also used in other studies, [(Li et al., 2021)](https://arxiv.org/pdf/2102.02079.pdf). We use a general partitioning strategy to set the number of labels that each client has; in the context of our study, this is either 1 or 2."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eFpLf8hLW4-o"
      },
      "source": [
        "### Statistical Resampling\n",
        "\n",
        "We use 10-fold Cross-Validation to sufficiently randomize the train/test split, to reduce sampling error. Further, we select 10 observations from each distribution setting to reduce selection bias. The data partitioning random seed is set to the current observation number. Therefore, we perform 100 repeats of the experiment per distribution setting."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dfjzxhM0boVb"
      },
      "source": [
        "## Test Bench Driver Code\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hnE2LjcqcuKX"
      },
      "outputs": [],
      "source": [
        "data_partitioning_strategies = [\"iid\", \"iid-quantity-skew\", \"noniid-distribution-based-label-imbalance\", \"noniid-quantity-based-label-imbalance-2\", \"noniid-quantity-based-label-imbalance-1\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4r71m9AWvI7K"
      },
      "outputs": [],
      "source": [
        "logging_folders = [\"log_29_03_2022__21_13_51_iid/\", \"log_29_03_2022__22_02_06_iid-quantity-skew/\", \"log_29_03_2022__22_02_56_noniid-distribution-based-label-imbalance/\", \"log_29_03_2022__23_14_51_noniid-quantity-based-label-imbalance-2/\", \"log_29_03_2022__23_15_21_noniid-quantity-based-label-imbalance-1/\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aZVUGttOv2HB"
      },
      "outputs": [],
      "source": [
        "use_data_sharing = [False, False, False, True, True]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZwVVSEG8zwEF"
      },
      "outputs": [],
      "source": [
        "classification_metrics = ['precision', 'recall', 'f1-score']\n",
        "aggregated_measures = ['value', 'std']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ByV1a6rHVtTH"
      },
      "outputs": [],
      "source": [
        "# Cross-validator\n",
        "skf = StratifiedKFold(n_splits=NUMBER_OF_VALIDATION_FOLDS, shuffle=True, random_state=42)\n",
        "\n",
        "current_repetition = 1\n",
        "\n",
        "for data_partitioning_strategy, logging_folder, data_sharing_flag in zip(data_partitioning_strategies, logging_folders, use_data_sharing):\n",
        "  # Overrides to global experiment config\n",
        "  DATA_PARTITIONING_STRATEGY = data_partitioning_strategy\n",
        "  LOG_DIR = LOGGING_FOLDERS_DIR + logging_folder\n",
        "  DATA_SHARING = data_sharing_flag\n",
        "\n",
        "  summed_federated_test_accs = np.zeros(NUMBER_OF_FL_ROUNDS)\n",
        "\n",
        "  aggregated_classification_macro_averages_of_locally_trained_models_on_local_data = {classification_metric: {aggregated_measure: [] \n",
        "                                                                                                              for aggregated_measure in aggregated_measures} \n",
        "                                                                                      for classification_metric in classification_metrics}\n",
        "  aggregated_classification_macro_averages_of_federated_model_on_local_data = {classification_metric: {aggregated_measure: [] \n",
        "                                                                                                       for aggregated_measure in aggregated_measures} \n",
        "                                                                               for classification_metric in classification_metrics}\n",
        "\n",
        "  for random_seed in range(0, NUMBER_OF_OBSERVATIONS_SELECTED_WITH_DATA_PARTITIONING_STRATEGY):\n",
        "\n",
        "    DATA_PARTITIONING_RANDOM_SEED = random_seed\n",
        "\n",
        "    summed_precision_recall_fscores_of_local_models_across_validation_folds = np.zeros((3, NUMBER_OF_CLIENTS))\n",
        "    summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds = np.zeros((3, NUMBER_OF_CLIENTS))\n",
        "    \n",
        "    for train_index, test_index in skf.split(X_, y):\n",
        "\n",
        "      print(f'Repetition number {current_repetition}...')\n",
        "\n",
        "      X_train, X_test = X_[train_index], X_[test_index]\n",
        "      y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n",
        "\n",
        "      if DATA_SHARING:\n",
        "        # Partition train set into client partition and holdout partition G, where β = 2.5%. Initialise with random seed for reproducability.\n",
        "        X_train_client, X_train_holdout, y_train_client, y_train_holdout = train_test_split(X_train, y_train, random_state=42, test_size=0.025, stratify=y_train)\n",
        "\n",
        "        # A warm-up model is trained on G\n",
        "        warm_up_model = classifier.fit(X_train_holdout, y_train_holdout)\n",
        "\n",
        "        # Create a random α portion of G to be merged with the data of each client, where α is 10%\n",
        "        X_train_shared, _, y_train_shared, _ = train_test_split(X_train_holdout, y_train_holdout, random_state=42, train_size=0.1, stratify=y_train_holdout)\n",
        "        \n",
        "      # Split train set into NUMBER_OF_CLIENTS partitions.\n",
        "      partitioned_train_set = get_train_partitions(X_train, y_train.to_numpy())\n",
        "\n",
        "      # Split test set into NUMBER_OF_CLIENTS partitions.\n",
        "      partitioned_test_set = get_test_partitions(X_test, y_test.to_numpy(), partitioned_train_set)\n",
        "\n",
        "      # Evaluate local models\n",
        "      local_models_y_preds = evaluate_local_models(partitioned_train_set, partitioned_test_set)\n",
        "\n",
        "      local_models_y_test = [y_test_partition for X_test_partition, y_test_partition in partitioned_test_set]\n",
        "\n",
        "      precision_recall_fscores_of_local_models = get_precision_recall_fscores_of_local_models(local_models_y_test, local_models_y_preds)\n",
        "\n",
        "      avg_local_model_precision_recall_fscore = get_mean_precision_recall_fscores_of_local_models(precision_recall_fscores_of_local_models)\n",
        "\n",
        "      measures_of_dispersion = get_measures_of_dispersion(precision_recall_fscores_of_local_models)\n",
        "\n",
        "      summed_precision_recall_fscores_of_local_models_across_validation_folds = np.add(summed_precision_recall_fscores_of_local_models_across_validation_folds, precision_recall_fscores_of_local_models)\n",
        "\n",
        "      for i, classification_metric in enumerate(classification_metrics):\n",
        "        aggregated_classification_macro_averages_of_locally_trained_models_on_local_data[classification_metric]['value'].append(avg_local_model_precision_recall_fscore[i])\n",
        "        aggregated_classification_macro_averages_of_locally_trained_models_on_local_data[classification_metric]['std'].append(measures_of_dispersion[i])\n",
        "\n",
        "\n",
        "      # Run sim with global config\n",
        "      sim_results = run_sim(partitioned_train_set, partitioned_test_set)\n",
        "\n",
        "      federated_y_preds = sim_results[\"federated_y_preds\"]\n",
        "\n",
        "      summed_federated_test_accs = np.add(summed_federated_test_accs, [mean(test_accs) for test_accs in zip(*sim_results[\"federated_test_accs\"])])\n",
        "\n",
        "      ultimate_precision_recall_fscores_of_federated_model_on_local_data = get_precision_recall_fscores_of_local_models(local_models_y_test, [client[-1] for client in federated_y_preds])\n",
        "\n",
        "      avg_ultimate_federated_model_precision_recall_fscore = get_mean_precision_recall_fscores_of_local_models(ultimate_precision_recall_fscores_of_federated_model_on_local_data)\n",
        "\n",
        "      measures_of_dispersion = get_measures_of_dispersion(ultimate_precision_recall_fscores_of_federated_model_on_local_data)\n",
        "\n",
        "      summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds = np.add(summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, ultimate_precision_recall_fscores_of_federated_model_on_local_data)\n",
        "      np.add(summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, ultimate_precision_recall_fscores_of_federated_model_on_local_data)\n",
        "\n",
        "\n",
        "      for i, classification_metric in enumerate(classification_metrics):\n",
        "        aggregated_classification_macro_averages_of_federated_model_on_local_data[classification_metric]['value'].append(avg_ultimate_federated_model_precision_recall_fscore[i])\n",
        "        aggregated_classification_macro_averages_of_federated_model_on_local_data[classification_metric]['std'].append(measures_of_dispersion[i])\n",
        "\n",
        "      current_repetition += 1\n",
        "\n",
        "    avg_precision_recall_fscores_of_local_models_across_validation_folds = np.divide(summed_precision_recall_fscores_of_local_models_across_validation_folds, NUMBER_OF_VALIDATION_FOLDS)\n",
        "    avg_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds = np.divide(summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, NUMBER_OF_VALIDATION_FOLDS)\n",
        "    np.divide(summed_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, NUMBER_OF_VALIDATION_FOLDS)\n",
        "\n",
        "    classification_macro_averages_filepath = LOG_DIR + DATA_PARTITIONING_STRATEGY + '_seed_' + str(DATA_PARTITIONING_RANDOM_SEED) + 'ultimate_precision_recall_fscores_of_models_across_10_validation_folds.json'\n",
        "    json.dump([{\"avg_precision_recall_fscores_of_local_models_across_validation_folds\": avg_precision_recall_fscores_of_local_models_across_validation_folds.tolist(), \n",
        "                \"stds\": np.std(avg_precision_recall_fscores_of_local_models_across_validation_folds, axis=1).tolist(), \n",
        "                \"means\": np.mean(avg_precision_recall_fscores_of_local_models_across_validation_folds, axis=1).tolist()}, \n",
        "           {\"avg_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds\": \n",
        "            avg_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds.tolist(), \n",
        "            \"stds\": np.std(avg_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, axis=1).tolist(), \n",
        "            \"means\": np.mean(avg_ultimate_precision_recall_fscores_of_federated_model_on_local_data_across_validation_folds, axis=1).tolist()}], open(classification_macro_averages_filepath, 'w'), indent=4)\n",
        "\n",
        "  avg_federated_test_accs = np.divide(summed_federated_test_accs, NUMBER_OF_REPEATS)\n",
        "\n",
        "  avg_classification_macro_averages_of_locally_trained_models_on_local_data = {classification_metric: {aggregated_measure: {'avg': np.mean(averages), '95% CI': [np.percentile(averages, 2.5), \n",
        "                                                                                                                                                                np.percentile(averages, 97.5)]} \n",
        "                                                                                                      for aggregated_measure, averages in aggregated_measures.items()} \n",
        "                                                                              for classification_metric, aggregated_measures in \n",
        "                                                                              aggregated_classification_macro_averages_of_locally_trained_models_on_local_data.items()}\n",
        "\n",
        "  avg_classification_macro_averages_of_federated_model_on_local_data = {classification_metric: {aggregated_measure: {'avg': np.mean(averages), '95% CI': [np.percentile(averages, 2.5), \n",
        "                                                                                                                                                          np.percentile(averages, 97.5)]} \n",
        "                                                                                                for aggregated_measure, averages in aggregated_measures.items()} \n",
        "                                                                        for classification_metric, aggregated_measures in \n",
        "                                                                        aggregated_classification_macro_averages_of_federated_model_on_local_data.items()}\n",
        "\n",
        "  federated_test_accs_filepath = LOG_DIR+f\"{DATA_PARTITIONING_STRATEGY}_federated_test_accs.json\"\n",
        "  json.dump({'federated_test_accs': avg_federated_test_accs.tolist()}, open(federated_test_accs_filepath, 'w'), indent=4)\n",
        "\n",
        "  classification_macro_averages_filepath = LOG_DIR + f'classification_macro_averages_federated_evaluation.json'\n",
        "  json.dump([{'avg_classification_averages_local_models': avg_classification_macro_averages_of_locally_trained_models_on_local_data}, \n",
        "            {'avg_classification_averages_federated_eval': avg_classification_macro_averages_of_federated_model_on_local_data}], open(classification_macro_averages_filepath, 'w'), indent=4)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "background_execution": "on",
      "collapsed_sections": [],
      "name": "Federated-Detection-Of-Hate-Speech.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}