oyez_decision_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0ecd86e2",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "c4d84fc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install pandas\n",
    "# !pip install numpy\n",
    "# !pip install sklearn\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.linear_model import Perceptron\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "\n",
    "# Just for visuals\n",
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17342c68",
   "metadata": {},
   "source": [
    "# 1. Dataset Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d815ec89",
   "metadata": {},
   "source": [
    "## Load Pre-processed Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "74897779",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 3464 cases.\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_pickle('res/task1_data.pkl')\n",
    "df.rename(columns={'Facts': 'facts'}, inplace=True)\n",
    "df.drop(columns=['index'], inplace=True)\n",
    "df.reset_index(inplace=True)\n",
    "\n",
    "print(f'There are {len(df)} cases.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "54de1f5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a mirrored case for each case, where the parties are swapped to prevent favoring first_party\n",
    "df_list = df.values.tolist()\n",
    "result = []\n",
    "for row in df_list:\n",
    "    result.append(row[1:])\n",
    "    mirrored_row = row.copy()\n",
    "    #  first_party is at index=4, second=5, winner_index=7\n",
    "    mirrored_row[4] = row[5]\n",
    "    mirrored_row[5] = row[4]\n",
    "    mirrored_row[7] = 1-mirrored_row[7]\n",
    "    result.append(mirrored_row[1:])\n",
    "df2 = pd.DataFrame(result)\n",
    "df2.rename(columns={\n",
    "    0: 'ID',\n",
    "    1: 'name',\n",
    "    2: 'href',\n",
    "    3: 'first_party',\n",
    "    4: 'second_party',\n",
    "    5: 'winning_party',\n",
    "    6: 'winner_index',\n",
    "    7: 'facts',\n",
    "}, inplace=True)\n",
    "df = df2\n",
    "df.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "761f95bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average facts character length: 1179\n",
      "Average facts word length: 189\n"
     ]
    }
   ],
   "source": [
    "avg_char = df['facts'].apply(lambda x: len(str(x))).mean()\n",
    "print(f'Average facts character length: {avg_char:.0f}')\n",
    "\n",
    "avg_word = df['facts'].apply(lambda x: len(str(x).split())).mean()\n",
    "print(f'Average facts word length: {avg_word:.0f}')\n",
    "\n",
    "del avg_char, avg_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "3059c8c0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>ID</th>\n",
       "      <th>name</th>\n",
       "      <th>href</th>\n",
       "      <th>first_party</th>\n",
       "      <th>second_party</th>\n",
       "      <th>winning_party</th>\n",
       "      <th>winner_index</th>\n",
       "      <th>facts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>50606</td>\n",
       "      <td>Roe v. Wade</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-18</td>\n",
       "      <td>Jane Roe</td>\n",
       "      <td>Henry Wade</td>\n",
       "      <td>Jane Roe</td>\n",
       "      <td>0</td>\n",
       "      <td>In 1970, Jane Roe (a fictional name used in court documents to protect the plaintiff’s identity) filed a lawsuit against Henry Wade, the district attorney of Dallas County, Texas, where she resided, challenging a Texas law making abortion illegal except by a doctor’s orders to save a woman’s life. In her lawsuit, Roe alleged that the state laws were unconstitutionally vague and abridged her right of personal privacy, protected by the First, Fourth, Fifth, Ninth, and Fourteenth Amendments.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>50606</td>\n",
       "      <td>Roe v. Wade</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-18</td>\n",
       "      <td>Henry Wade</td>\n",
       "      <td>Jane Roe</td>\n",
       "      <td>Jane Roe</td>\n",
       "      <td>1</td>\n",
       "      <td>In 1970, Jane Roe (a fictional name used in court documents to protect the plaintiff’s identity) filed a lawsuit against Henry Wade, the district attorney of Dallas County, Texas, where she resided, challenging a Texas law making abortion illegal except by a doctor’s orders to save a woman’s life. In her lawsuit, Roe alleged that the state laws were unconstitutionally vague and abridged her right of personal privacy, protected by the First, Fourth, Fifth, Ninth, and Fourteenth Amendments.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>50613</td>\n",
       "      <td>Stanley v. Illinois</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-5014</td>\n",
       "      <td>Peter Stanley, Sr.</td>\n",
       "      <td>Illinois</td>\n",
       "      <td>Stanley</td>\n",
       "      <td>0</td>\n",
       "      <td>Joan Stanley had three children with Peter Stanley. The Stanleys never married, but lived together off and on for 18 years. When Joan died, the State of Illinois took the children. Under Illinois law, unwed fathers were presumed unfit parents regardless of their actual fitness and their children became wards of the state. Peter appealed the decision, arguing that the Illinois law violated the Equal Protection Clause of the Fourteenth Amendment because unwed mothers were not deprived of their children without a showing that they were actually unfit parents. The Illinois Supreme Court rejected Stanley’s Equal Protection claim, holding that his actual fitness as a parent was irrelevant because he and the children’s mother were unmarried.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index     ID                 name                                     href  \\\n",
       "0      0  50606          Roe v. Wade    https://api.oyez.org/cases/1971/70-18   \n",
       "1      1  50606          Roe v. Wade    https://api.oyez.org/cases/1971/70-18   \n",
       "2      2  50613  Stanley v. Illinois  https://api.oyez.org/cases/1971/70-5014   \n",
       "\n",
       "           first_party second_party winning_party  winner_index  \\\n",
       "0             Jane Roe   Henry Wade      Jane Roe             0   \n",
       "1           Henry Wade     Jane Roe      Jane Roe             1   \n",
       "2  Peter Stanley, Sr.      Illinois       Stanley             0   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      facts  \n",
       "0                                                                                                                                                                                                                                                             In 1970, Jane Roe (a fictional name used in court documents to protect the plaintiff’s identity) filed a lawsuit against Henry Wade, the district attorney of Dallas County, Texas, where she resided, challenging a Texas law making abortion illegal except by a doctor’s orders to save a woman’s life. In her lawsuit, Roe alleged that the state laws were unconstitutionally vague and abridged her right of personal privacy, protected by the First, Fourth, Fifth, Ninth, and Fourteenth Amendments.  \n",
       "1                                                                                                                                                                                                                                                             In 1970, Jane Roe (a fictional name used in court documents to protect the plaintiff’s identity) filed a lawsuit against Henry Wade, the district attorney of Dallas County, Texas, where she resided, challenging a Texas law making abortion illegal except by a doctor’s orders to save a woman’s life. In her lawsuit, Roe alleged that the state laws were unconstitutionally vague and abridged her right of personal privacy, protected by the First, Fourth, Fifth, Ninth, and Fourteenth Amendments.  \n",
       "2  Joan Stanley had three children with Peter Stanley. The Stanleys never married, but lived together off and on for 18 years. When Joan died, the State of Illinois took the children. Under Illinois law, unwed fathers were presumed unfit parents regardless of their actual fitness and their children became wards of the state. Peter appealed the decision, arguing that the Illinois law violated the Equal Protection Clause of the Fourteenth Amendment because unwed mothers were not deprived of their children without a showing that they were actually unfit parents. The Illinois Supreme Court rejected Stanley’s Equal Protection claim, holding that his actual fitness as a parent was irrelevant because he and the children’s mother were unmarried.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(df.head(n=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6578cedd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 6928 cases.\n",
      "There are 3464 rows for class 0.\n",
      "There are 3464 rows for class 1.\n"
     ]
    }
   ],
   "source": [
    "print(f'There are {len(df)} cases.')\n",
    "print(f'There are {len(df[df[\"winner_index\"]==0])} rows for class 0.')\n",
    "print(f'There are {len(df[df[\"winner_index\"]==1])} rows for class 1.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "f046dc86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    6928.000000\n",
       "mean     1179.302252\n",
       "std       556.295521\n",
       "min        95.000000\n",
       "25%       784.000000\n",
       "50%      1112.500000\n",
       "75%      1496.000000\n",
       "max      6108.000000\n",
       "Name: facts, dtype: float64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Facts character stats\n",
    "df['facts'].apply(lambda x: len(str(x))).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "ac4ed6e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    6928.000000\n",
       "mean      188.618938\n",
       "std        91.490377\n",
       "min        13.000000\n",
       "25%       125.000000\n",
       "50%       176.000000\n",
       "75%       239.000000\n",
       "max       974.000000\n",
       "Name: facts, dtype: float64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Facts word stats\n",
    "df['facts'].apply(lambda x: len(str(x).split())).describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41f4afca",
   "metadata": {},
   "source": [
    "# 2. AutoJudge\n",
    "Given the neutral absolute facts, predict the decision of the case."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e242ac0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Perform an 80-20 split for training and testing data\n",
    "X_train_party1_text, X_test_party1_text, \\\n",
    "X_train_party2_text, X_test_party2_text, \\\n",
    "X_train_facts_text, X_test_facts_text, \\\n",
    "y_train, y_test = train_test_split(\n",
    "    df['first_party'],\n",
    "    df['second_party'],\n",
    "    df['facts'],\n",
    "    df['winner_index'],\n",
    "    test_size=0.2,\n",
    "    stratify=df['winner_index']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dea09964",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TF-IDF Feature Extraction\n",
    "vectorizer = TfidfVectorizer()\n",
    "X_train_facts = vectorizer.fit_transform(X_train_facts_text)\n",
    "X_test_facts = vectorizer.transform(X_test_facts_text)\n",
    "X_train_party1 = vectorizer.transform(X_train_party1_text)\n",
    "X_test_party1 = vectorizer.transform(X_test_party1_text)\n",
    "X_train_party2 = vectorizer.transform(X_train_party2_text)\n",
    "X_test_party2 = vectorizer.transform(X_test_party2_text)\n",
    "\n",
    "X_train = np.concatenate([X_train_party1.todense(), X_train_party2.todense(), X_train_facts.todense()], axis=1)\n",
    "X_test = np.concatenate([X_test_party1.todense(), X_test_party2.todense(), X_test_facts.todense()], axis=1)\n",
    "\n",
    "del X_train_facts, X_train_party1, X_train_party2\n",
    "del X_test_facts, X_test_party1, X_test_party2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62910aba",
   "metadata": {},
   "source": [
    "## Perceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "f22519ef",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Perceptron - Train\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.80      0.79      2771\n",
      "           1       0.80      0.76      0.78      2771\n",
      "\n",
      "    accuracy                           0.78      5542\n",
      "   macro avg       0.78      0.78      0.78      5542\n",
      "weighted avg       0.78      0.78      0.78      5542\n",
      "\n",
      "Perceptron - Test\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.69      0.66       693\n",
      "           1       0.66      0.61      0.63       693\n",
      "\n",
      "    accuracy                           0.65      1386\n",
      "   macro avg       0.65      0.65      0.65      1386\n",
      "weighted avg       0.65      0.65      0.65      1386\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Perceptron\n",
    "model_perceptron = Perceptron(\n",
    "    alpha=0.0001,\n",
    "    max_iter=5,\n",
    "    n_iter_no_change=5,\n",
    "    penalty='l1',\n",
    "    tol=1e-3,\n",
    "    validation_fraction=0.1\n",
    ")\n",
    "model_perceptron.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = model_perceptron.predict(X_train)\n",
    "y_test_pred = model_perceptron.predict(X_test)\n",
    "\n",
    "print('Perceptron - Train\\n', classification_report(y_train, y_train_pred, zero_division=0))\n",
    "print('Perceptron - Test\\n', classification_report(y_test, y_test_pred, zero_division=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17d29c35",
   "metadata": {},
   "source": [
    "## SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "a5ce4c5e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM - Train\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.91      0.91      2771\n",
      "           1       0.91      0.91      0.91      2771\n",
      "\n",
      "    accuracy                           0.91      5542\n",
      "   macro avg       0.91      0.91      0.91      5542\n",
      "weighted avg       0.91      0.91      0.91      5542\n",
      "\n",
      "SVM - Test\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.60      0.61      0.61       693\n",
      "           1       0.60      0.60      0.60       693\n",
      "\n",
      "    accuracy                           0.60      1386\n",
      "   macro avg       0.60      0.60      0.60      1386\n",
      "weighted avg       0.60      0.60      0.60      1386\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# SVM\n",
    "model_svm = LinearSVC(\n",
    "    max_iter=5,\n",
    "    C=0.1,\n",
    "    intercept_scaling=0.1,\n",
    "    loss='squared_hinge',\n",
    "    penalty='l2',\n",
    "    tol=1e-2\n",
    ")\n",
    "model_svm.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = model_svm.predict(X_train)\n",
    "y_test_pred = model_svm.predict(X_test)\n",
    "\n",
    "print('SVM - Train\\n', classification_report(y_train, y_train_pred, zero_division=0))\n",
    "print('SVM - Test\\n', classification_report(y_test, y_test_pred, zero_division=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a028ab17",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "3e860a56",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression - Train\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.92      0.92      2771\n",
      "           1       0.92      0.92      0.92      2771\n",
      "\n",
      "    accuracy                           0.92      5542\n",
      "   macro avg       0.92      0.92      0.92      5542\n",
      "weighted avg       0.92      0.92      0.92      5542\n",
      "\n",
      "Logistic Regression - Test\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.61      0.61      0.61       693\n",
      "           1       0.61      0.60      0.61       693\n",
      "\n",
      "    accuracy                           0.61      1386\n",
      "   macro avg       0.61      0.61      0.61      1386\n",
      "weighted avg       0.61      0.61      0.61      1386\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Logistic Regression\n",
    "model_log_reg = LogisticRegression()\n",
    "model_log_reg.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = model_log_reg.predict(X_train)\n",
    "y_test_pred = model_log_reg.predict(X_test)\n",
    "\n",
    "print('Logistic Regression - Train\\n', classification_report(y_train, y_train_pred, zero_division=0))\n",
    "print('Logistic Regression - Test\\n', classification_report(y_test, y_test_pred, zero_division=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0249a2a7",
   "metadata": {},
   "source": [
    "## Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "4f5eb3d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes - Train\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.85      0.84      2771\n",
      "           1       0.85      0.84      0.84      2771\n",
      "\n",
      "    accuracy                           0.84      5542\n",
      "   macro avg       0.84      0.84      0.84      5542\n",
      "weighted avg       0.84      0.84      0.84      5542\n",
      "\n",
      "Naive Bayes - Test\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.59      0.61      0.60       693\n",
      "           1       0.60      0.58      0.59       693\n",
      "\n",
      "    accuracy                           0.60      1386\n",
      "   macro avg       0.60      0.60      0.60      1386\n",
      "weighted avg       0.60      0.60      0.60      1386\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Naive Bayes\n",
    "model_naive_bayes = MultinomialNB(\n",
    "    alpha=3\n",
    ")\n",
    "model_naive_bayes.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = model_naive_bayes.predict(X_train)\n",
    "y_test_pred = model_naive_bayes.predict(X_test)\n",
    "\n",
    "print('Naive Bayes - Train\\n', classification_report(y_train, y_train_pred, zero_division=0))\n",
    "print('Naive Bayes - Test\\n', classification_report(y_test, y_test_pred, zero_division=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e228dfb",
   "metadata": {},
   "source": [
    "## Multi-layer Perceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "0306c82e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MLP - Train\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.93      0.91      2771\n",
      "           1       0.92      0.89      0.90      2771\n",
      "\n",
      "    accuracy                           0.91      5542\n",
      "   macro avg       0.91      0.91      0.91      5542\n",
      "weighted avg       0.91      0.91      0.91      5542\n",
      "\n",
      "MLP - Test\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.71      0.67       693\n",
      "           1       0.67      0.60      0.64       693\n",
      "\n",
      "    accuracy                           0.66      1386\n",
      "   macro avg       0.66      0.66      0.65      1386\n",
      "weighted avg       0.66      0.66      0.65      1386\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# MLP\n",
    "model_mlp = MLPClassifier(\n",
    "    early_stopping=True,\n",
    "    beta_2=0,\n",
    "    max_iter=10,\n",
    ")\n",
    "model_mlp.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = model_mlp.predict(X_train)\n",
    "y_test_pred = model_mlp.predict(X_test)\n",
    "\n",
    "print('MLP - Train\\n', classification_report(y_train, y_train_pred, zero_division=0))\n",
    "print('MLP - Test\\n', classification_report(y_test, y_test_pred, zero_division=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e675dbd",
   "metadata": {},
   "source": [
    "## K-Nearest Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "76129d75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN - Train\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      2771\n",
      "           1       1.00      1.00      1.00      2771\n",
      "\n",
      "    accuracy                           1.00      5542\n",
      "   macro avg       1.00      1.00      1.00      5542\n",
      "weighted avg       1.00      1.00      1.00      5542\n",
      "\n",
      "KNN - Test\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.63      0.83      0.72       693\n",
      "           1       0.76      0.52      0.61       693\n",
      "\n",
      "    accuracy                           0.68      1386\n",
      "   macro avg       0.69      0.68      0.67      1386\n",
      "weighted avg       0.69      0.68      0.67      1386\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# KNN\n",
    "model_knn = KNeighborsClassifier(n_neighbors=3, weights='distance')\n",
    "model_knn.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = model_knn.predict(X_train)\n",
    "y_test_pred = model_knn.predict(X_test)\n",
    "\n",
    "print('KNN - Train\\n', classification_report(y_train, y_train_pred, zero_division=0))\n",
    "print('KNN - Test\\n', classification_report(y_test, y_test_pred, zero_division=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2005698f",
   "metadata": {},
   "source": [
    "## Calibrated Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3418ebd5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calibrated Classifier - Train\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.93      0.94      2771\n",
      "           1       0.93      0.94      0.94      2771\n",
      "\n",
      "    accuracy                           0.94      5542\n",
      "   macro avg       0.94      0.94      0.94      5542\n",
      "weighted avg       0.94      0.94      0.94      5542\n",
      "\n",
      "Calibrated Classifier - Test\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.66      0.51      0.57       693\n",
      "           1       0.60      0.73      0.66       693\n",
      "\n",
      "    accuracy                           0.62      1386\n",
      "   macro avg       0.63      0.62      0.62      1386\n",
      "weighted avg       0.63      0.62      0.62      1386\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Calibrated Classifier\n",
    "model_calibrated_classifier = CalibratedClassifierCV(\n",
    "    method='isotonic'\n",
    ")\n",
    "model_calibrated_classifier.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = model_calibrated_classifier.predict(X_train)\n",
    "y_test_pred = model_calibrated_classifier.predict(X_test)\n",
    "\n",
    "print('Calibrated Classifier - Train\\n', classification_report(y_train, y_train_pred, zero_division=0))\n",
    "print('Calibrated Classifier - Test\\n', classification_report(y_test, y_test_pred, zero_division=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd3da5ba",
   "metadata": {},
   "source": [
    "## Create Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "bc8cf6ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = []\n",
    "models.append(('perceptron', model_perceptron))\n",
    "models.append(('svm', model_svm))\n",
    "models.append(('logistic_regression', model_log_reg))\n",
    "models.append(('naive_bayes', model_naive_bayes))\n",
    "models.append(('multi_layer_perceptron', model_mlp))\n",
    "models.append(('k_nearest_neighbors', model_knn))\n",
    "models.append(('calibrated_classifier', model_calibrated_classifier))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "20ce7b4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ensemble - Train\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.94      0.95      2771\n",
      "           1       0.94      0.96      0.95      2771\n",
      "\n",
      "    accuracy                           0.95      5542\n",
      "   macro avg       0.95      0.95      0.95      5542\n",
      "weighted avg       0.95      0.95      0.95      5542\n",
      "\n",
      "Ensemble - Test\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.61      0.63       693\n",
      "           1       0.63      0.67      0.65       693\n",
      "\n",
      "    accuracy                           0.64      1386\n",
      "   macro avg       0.64      0.64      0.64      1386\n",
      "weighted avg       0.64      0.64      0.64      1386\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Ensemble\n",
    "ensemble = VotingClassifier(models, voting='hard')\n",
    "ensemble.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = ensemble.predict(X_train)\n",
    "y_test_pred = ensemble.predict(X_test)\n",
    "\n",
    "print('Ensemble - Train\\n', classification_report(y_train, y_train_pred, zero_division=0))\n",
    "print('Ensemble - Test\\n', classification_report(y_test, y_test_pred, zero_division=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "18cb74a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VotingClassifier(estimators=[('perceptron',\n",
      "                              Perceptron(max_iter=5, n_iter_no_change=10,\n",
      "                                         penalty='l1', tol=1e-05,\n",
      "                                         validation_fraction=0.3)),\n",
      "                             ('svm',\n",
      "                              LinearSVC(C=3, intercept_scaling=0.1,\n",
      "                                        loss='hinge', max_iter=5, tol=0.01)),\n",
      "                             ('logistic_regression', LogisticRegression()),\n",
      "                             ('naive_bayes', MultinomialNB(alpha=2)),\n",
      "                             ('multi_layer_perceptron',\n",
      "                              MLPClassifier(beta_1=0.99, beta_2=0,\n",
      "                                            early_stopping=True, epsilon=1e-05,\n",
      "                                            learning_rate_init=0.01,\n",
      "                                            max_iter=10, n_iter_no_change=20,\n",
      "                                            tol=0.01)),\n",
      "                             ('k_nearest_neighbors',\n",
      "                              KNeighborsClassifier(n_neighbors=3,\n",
      "                                                   weights='distance')),\n",
      "                             ('calibrated_classifier',\n",
      "                              CalibratedClassifierCV(method='isotonic'))])\n"
     ]
    }
   ],
   "source": [
    "print(ensemble)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f38705d",
   "metadata": {},
   "source": [
    "## Demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4c858879",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(party1, party2, facts):\n",
    "    X_party1 = vectorizer.transform([party1]).todense()\n",
    "    X_party2 = vectorizer.transform([party2]).todense()\n",
    "    X_facts = vectorizer.transform([facts]).todense()\n",
    "    X = np.concatenate([X_party1, X_party2, X_facts], axis=1)\n",
    "    return ensemble.predict(X)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "cd9c6b1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expecting 1 but got 1\n"
     ]
    }
   ],
   "source": [
    "out = predict('Jake', 'John', 'John was assaulted by Jake at gun point. He bled severely while Jake escaped the crime scene. The entire footage was captured by a CCTV of a nearby gas station.')\n",
    "print(f'Expecting 1 but got {out}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "cc6f2e70",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expecting 0 but got 0\n"
     ]
    }
   ],
   "source": [
    "out = predict('The Florida Bar', 'Lanell Williams-Yulee', \"\"\"During her candidacy for County Court Judge in Hillsborough County, Florida, Lanell Williams-Yulee personally solicited campaign contributions. She stated that she served as the \"community Public Defender\" – although her title was \"assistant public defender\" – and inaccurately stated in the media that there was no incumbent in the judicial race for which she was running. The Florida Bar filed a complaint against Williams-Yulee and alleged that her actions during the campaign violated the rules regulating The Florida Bar. A referee was appointed who suggested that Williams-Yulee receive a public reprimand. Williams-Yulee appealed the referee's finding, and the Supreme Court of Florida held that Williams-Yulee violated bar rules for directly soliciting funds for her judicial campaign. Williams-Yulee appealed and claimed that The Florida Bar rule prohibiting a candidate from personal solicitation of funds violated the First Amendment protection of freedom of speech.\"\"\")\n",
    "print(f'Expecting 0 but got {out}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "05458fe4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expecting 1 but got 1\n"
     ]
    }
   ],
   "source": [
    "out = predict('Rob Bonta, Attorney General of California', 'Americans for Prosperity Foundation', \"\"\"The California Attorney General’s office has a policy requiring charities to provide the state, on a confidential basis, information about their major donors, purportedly to help the state protect consumers from fraud and the misuse of their charitable contributions. Petitioner Americans for Prosperity (and the petitioner in the consolidated case, Thomas More Law Center) either failed to file or filed redacted lists of their major donors with the California Attorney General’s office, despite filing complete lists with the federal Internal Revenue Service, as required by federal law. In response to demands by the California Attorney General that they file the lists, the organizations filed a lawsuit alleging that the filing requirement unconstitutionally burdened their First Amendment right to free association by deterring individuals from financially supporting them. The organizations provided evidence that although the state is required to keep donor names private, the state’s database was vulnerable to hacking, and many donor names were repeatedly released to the public. Based in part on this finding, the district court granted both organizations’ motions for a preliminary injunction and then ultimately found for them after a trial, holding that the organizations and their donors were entitled to First Amendment protection under the principles established in the Supreme Court’s decision in NAACP v. Alabama. In so holding, the court reasoned that the government’s filing demands were not the “least restrictive means” of obtaining the information and thus did not satisfy “strict scrutiny.” A panel of the you.S. Court of Appeals for the Ninth Circuit reversed, based on its conclusion that “exacting scrutiny” rather than “strict scrutiny” was the appropriate standard, and “exacting scrutiny” requires that the government show that the disclosure and reporting requirements are justified by a compelling government interest and that the legislation is narrowly tailored to serve that interest. The Ninth Circuit denied the petition for a rehearing en banc.\"\"\")\n",
    "print(f'Expecting 1 but got {out}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "7d65ebbf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expecting 0 but got 1\n"
     ]
    }
   ],
   "source": [
    "out = predict('Max', 'University of Washington Law School', \"\"\"Max was denied admission to the University of Washington Law School despite test scores that were higher than some of the minorities admitted. Max then successfully asked a trial court to require the school to admit him. On appeal, the Washington Supreme Court reversed, upholding the school's decision to deny Max admission. The you.S. Supreme Court considered the case as Max was entering his final year of school.\"\"\")\n",
    "print(f'Expecting 0 but got {out}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}