oyez_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f8993439",
   "metadata": {},
   "source": [
    "# Filter Dataset from oyez_gather script\n",
    "This script takes in input oyez.json as generated by the oyez_gather script.\n",
    "Here's an already generated dataset:\n",
    "\n",
    "Minimal JSON compact form (216MB):\n",
    "https://www.dropbox.com/s/9kyk0dr2gf3ls23/oyez.json?dl=0\n",
    "\n",
    "Prettified JSON human-readable form (431 MB):\n",
    "https://www.dropbox.com/s/52a58aac8iujupv/oyez_pretty.json?dl=0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ecd86e2",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c4d84fc5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\Sanavesa\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Sanavesa\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# !pip install json\n",
    "# !pip install pandas\n",
    "# !pip install numpy\n",
    "# !pip install nltk\n",
    "# !pip install bs4\n",
    "# !pip install re\n",
    "# !pip install contractions\n",
    "\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "nltk.download('wordnet')\n",
    "nltk.download('punkt')\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import contractions\n",
    "from collections import defaultdict\n",
    "import string\n",
    "import difflib\n",
    "import os.path\n",
    "import pickle\n",
    "\n",
    "# Just for visuals\n",
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17342c68",
   "metadata": {},
   "source": [
    "# 1. Dataset Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40ff3a44",
   "metadata": {},
   "source": [
    "## Load the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7cc15010",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../oyez.json', 'r') as f:\n",
    "    data = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5ed3f1a",
   "metadata": {},
   "source": [
    "## Filter the dataset to only the columns we need with cases with non-missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c2c9e55a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns true if the given case's judgment has been decided, false otherwise\n",
    "def is_decided(entry):\n",
    "    for timeline in entry['timeline']:\n",
    "        if timeline['event'] == \"Decided\":\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "# Returns true if the given case has the necessary columns for our analysis, false otherwise\n",
    "# Currently only considers:\n",
    "#    1- case's judgment has been decided\n",
    "#    2- case has a non-empty facts field\n",
    "#    3- case has a non-empty decisions field\n",
    "#    4- case with the same first/second party\n",
    "#    5- case has facts field with 4 or more words\n",
    "#    6- case has a winning party\n",
    "def is_entry_complete(entry):\n",
    "    try:\n",
    "        if not is_decided(entry):\n",
    "            return False\n",
    "        facts = entry['facts_of_the_case']\n",
    "        if facts == None or len(str(facts)) == 0:\n",
    "            return False\n",
    "        # Ignore case with facts being Currently unknown, Not Available, Currently unavailable.\n",
    "        if len(str(facts).split()) <= 3:\n",
    "            return False\n",
    "        if entry['decisions'] == None or len(entry['decisions']) == 0:\n",
    "            return False\n",
    "        if entry['first_party'] == entry['second_party'] and entry['first_party'] != None:\n",
    "            return False\n",
    "        return True\n",
    "    except:\n",
    "        return False\n",
    "\n",
    "# Returns a dict with only the necessary columns we need from a given case\n",
    "# Currently the followings columns are considered:\n",
    "#    1-  case ID (assigned by oyez.org)\n",
    "#    2-  case name\n",
    "#    3-  href URL to the oyez.org case\n",
    "#    4-  name of the first party\n",
    "#    5-  name of the second party\n",
    "#    6-  facts of the case\n",
    "#    7-  name of the winning party\n",
    "def filter_entry(entry):\n",
    "    row = {}\n",
    "    row['ID'] = entry['ID']\n",
    "    row['name'] = entry['name']\n",
    "    row['href'] = entry['href']\n",
    "    row['first_party'] = entry['first_party']\n",
    "    row['second_party'] = entry['second_party']\n",
    "    row['winning_party'] = None\n",
    "    for decision in entry['decisions']:\n",
    "        winning_party = decision['winning_party']\n",
    "        if winning_party != None and len(winning_party) > 0 and str(winning_party).lower() not in ['dismissal', 'n/a']:\n",
    "            row['winning_party'] = winning_party\n",
    "            break\n",
    "    row['facts'] = entry['facts_of_the_case']\n",
    "    return row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e5cd31bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_data = []\n",
    "for entry in data:\n",
    "    if is_entry_complete(entry):\n",
    "        filtered_data.append(filter_entry(entry))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d815ec89",
   "metadata": {},
   "source": [
    "## Prepare Pandas DataFrame for the Dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "74897779",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 2384 cases.\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(filtered_data)\n",
    "\n",
    "# Remove rows with missing values in the listed columns\n",
    "df.dropna(inplace=True, subset=['first_party', 'second_party', 'facts', 'winning_party'], how='any')\n",
    "df.reset_index(inplace=True)\n",
    "\n",
    "print(f'There are {len(df)} cases.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3059c8c0",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>ID</th>\n",
       "      <th>name</th>\n",
       "      <th>href</th>\n",
       "      <th>first_party</th>\n",
       "      <th>second_party</th>\n",
       "      <th>winning_party</th>\n",
       "      <th>facts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>50606</td>\n",
       "      <td>Roe v. Wade</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-18</td>\n",
       "      <td>Jane Roe</td>\n",
       "      <td>Henry Wade</td>\n",
       "      <td>Jane Roe</td>\n",
       "      <td>&lt;p&gt;In 1970, Jane Roe (a fictional name used in court documents to protect the plaintiff’s identity) filed a lawsuit against Henry Wade, the district attorney of Dallas County, Texas, where she resided, challenging a Texas law making abortion illegal except by a doctor’s orders to save a woman’s life. In her lawsuit, Roe alleged that the state laws were unconstitutionally vague and abridged her right of personal privacy, protected by the First, Fourth, Fifth, Ninth, and Fourteenth Amendments.&lt;/p&gt;\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>50613</td>\n",
       "      <td>Stanley v. Illinois</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-5014</td>\n",
       "      <td>Peter Stanley, Sr.</td>\n",
       "      <td>Illinois</td>\n",
       "      <td>Stanley</td>\n",
       "      <td>&lt;p&gt;Joan Stanley had three children with Peter Stanley.  The Stanleys never married, but lived together off and on for 18 years.  When Joan died, the State of Illinois took the children.  Under Illinois law, unwed fathers were presumed unfit parents regardless of their actual fitness and their children became wards of the state.  Peter appealed the decision, arguing that the Illinois law violated the Equal Protection Clause of the Fourteenth Amendment because unwed mothers were not deprived of their children without a showing that they were actually unfit parents.  The Illinois Supreme Court rejected Stanley’s Equal Protection claim, holding that his actual fitness as a parent was irrelevant because he and the children’s mother were unmarried.&lt;/p&gt;\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>50623</td>\n",
       "      <td>Giglio v. United States</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-29</td>\n",
       "      <td>John Giglio</td>\n",
       "      <td>United States</td>\n",
       "      <td>Giglio</td>\n",
       "      <td>&lt;p&gt;John Giglio was convicted of passing forged money orders.  While his appeal to the U.S. Court of Appeals for the Second Circuit was pending, Giglio’s counsel discovered new evidence. The evidence indicated that the prosecution failed to disclose that it promised a key witness immunity from prosecution in exchange for testimony against Giglio.  The district court denied Giglio’s motion for a new trial, finding that the error did not affect the verdict.  The Court of Appeals affirmed.&lt;/p&gt;\\n</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index     ID                     name  \\\n",
       "0      0  50606              Roe v. Wade   \n",
       "1      1  50613      Stanley v. Illinois   \n",
       "2      2  50623  Giglio v. United States   \n",
       "\n",
       "                                      href          first_party  \\\n",
       "0    https://api.oyez.org/cases/1971/70-18             Jane Roe   \n",
       "1  https://api.oyez.org/cases/1971/70-5014  Peter Stanley, Sr.    \n",
       "2    https://api.oyez.org/cases/1971/70-29         John Giglio    \n",
       "\n",
       "    second_party winning_party  \\\n",
       "0     Henry Wade      Jane Roe   \n",
       "1       Illinois       Stanley   \n",
       "2  United States        Giglio   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    facts  \n",
       "0                                                                                                                                                                                                                                                                  <p>In 1970, Jane Roe (a fictional name used in court documents to protect the plaintiff’s identity) filed a lawsuit against Henry Wade, the district attorney of Dallas County, Texas, where she resided, challenging a Texas law making abortion illegal except by a doctor’s orders to save a woman’s life. In her lawsuit, Roe alleged that the state laws were unconstitutionally vague and abridged her right of personal privacy, protected by the First, Fourth, Fifth, Ninth, and Fourteenth Amendments.</p>\\n  \n",
       "1  <p>Joan Stanley had three children with Peter Stanley.  The Stanleys never married, but lived together off and on for 18 years.  When Joan died, the State of Illinois took the children.  Under Illinois law, unwed fathers were presumed unfit parents regardless of their actual fitness and their children became wards of the state.  Peter appealed the decision, arguing that the Illinois law violated the Equal Protection Clause of the Fourteenth Amendment because unwed mothers were not deprived of their children without a showing that they were actually unfit parents.  The Illinois Supreme Court rejected Stanley’s Equal Protection claim, holding that his actual fitness as a parent was irrelevant because he and the children’s mother were unmarried.</p>\\n  \n",
       "2                                                                                                                                                                                                                                                                        <p>John Giglio was convicted of passing forged money orders.  While his appeal to the U.S. Court of Appeals for the Second Circuit was pending, Giglio’s counsel discovered new evidence. The evidence indicated that the prosecution failed to disclose that it promised a key witness immunity from prosecution in exchange for testimony against Giglio.  The district court denied Giglio’s motion for a new trial, finding that the error did not affect the verdict.  The Court of Appeals affirmed.</p>\\n  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(df.head(n=3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3f7f3f5",
   "metadata": {},
   "source": [
    "# 2. Preprocess Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbeec309",
   "metadata": {},
   "source": [
    "## Statistics before Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cbc0fcd9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average facts character length (before preprocesesing): 1142\n",
      "Average facts word length (before preprocesesing): 177\n"
     ]
    }
   ],
   "source": [
    "avg_char_before_preprocessing = df['facts'].apply(lambda x: len(str(x))).mean()\n",
    "print(f'Average facts character length (before preprocesesing): {avg_char_before_preprocessing:.0f}')\n",
    "\n",
    "avg_word_before_preprocessing = df['facts'].apply(lambda x: len(str(x).split())).mean()\n",
    "print(f'Average facts word length (before preprocesesing): {avg_word_before_preprocessing:.0f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54303c9c",
   "metadata": {},
   "source": [
    "## Remove HTML/URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e3945388",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to remove HTML tags and URLs from a string\n",
    "def sanitize_review(text):\n",
    "    # remove HTML tags\n",
    "    text = BeautifulSoup(str(text), 'html.parser').get_text()   \n",
    "    # remove URLS\n",
    "    text = re.sub(r'http\\S+', '', str(text))\n",
    "    return text\n",
    "\n",
    "df['facts'] = df['facts'].apply(sanitize_review)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5215a79a",
   "metadata": {},
   "source": [
    "## Remove non-UTF8 characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0c5cf120",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_non_ascii(text):\n",
    "    return text.encode(encoding='utf-8', errors='ignore').decode()\n",
    "\n",
    "df['facts'] = df['facts'].apply(remove_non_ascii)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74009061",
   "metadata": {},
   "source": [
    "## Remove Contractions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4d8fd034",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_contractions(text):\n",
    "    return contractions.fix(text)\n",
    "\n",
    "df['facts'] = df['facts'].apply(fix_contractions)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02c2d53c",
   "metadata": {},
   "source": [
    "## Remove extra spaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bcf32717",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_extra_spaces(text):\n",
    "    return ' '.join(str(text).split())\n",
    "\n",
    "df['facts'] = df['facts'].apply(remove_extra_spaces)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97a21e92",
   "metadata": {},
   "source": [
    "## Statistics after Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "761f95bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average facts character length (after preprocesesing): 1126\n",
      "Average facts word length (after preprocesesing): 177\n"
     ]
    }
   ],
   "source": [
    "avg_char_after_preprocessing = df['facts'].apply(lambda x: len(str(x))).mean()\n",
    "print(f'Average facts character length (after preprocesesing): {avg_char_after_preprocessing:.0f}')\n",
    "\n",
    "avg_word_after_preprocessing = df['facts'].apply(lambda x: len(str(x).split())).mean()\n",
    "print(f'Average facts word length (after preprocesesing): {avg_word_after_preprocessing:.0f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b2efc36a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>ID</th>\n",
       "      <th>name</th>\n",
       "      <th>href</th>\n",
       "      <th>first_party</th>\n",
       "      <th>second_party</th>\n",
       "      <th>winning_party</th>\n",
       "      <th>facts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>50606</td>\n",
       "      <td>Roe v. Wade</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-18</td>\n",
       "      <td>Jane Roe</td>\n",
       "      <td>Henry Wade</td>\n",
       "      <td>Jane Roe</td>\n",
       "      <td>In 1970, Jane Roe (a fictional name used in court documents to protect the plaintiff’s identity) filed a lawsuit against Henry Wade, the district attorney of Dallas County, Texas, where she resided, challenging a Texas law making abortion illegal except by a doctor’s orders to save a woman’s life. In her lawsuit, Roe alleged that the state laws were unconstitutionally vague and abridged her right of personal privacy, protected by the First, Fourth, Fifth, Ninth, and Fourteenth Amendments.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>50613</td>\n",
       "      <td>Stanley v. Illinois</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-5014</td>\n",
       "      <td>Peter Stanley, Sr.</td>\n",
       "      <td>Illinois</td>\n",
       "      <td>Stanley</td>\n",
       "      <td>Joan Stanley had three children with Peter Stanley. The Stanleys never married, but lived together off and on for 18 years. When Joan died, the State of Illinois took the children. Under Illinois law, unwed fathers were presumed unfit parents regardless of their actual fitness and their children became wards of the state. Peter appealed the decision, arguing that the Illinois law violated the Equal Protection Clause of the Fourteenth Amendment because unwed mothers were not deprived of their children without a showing that they were actually unfit parents. The Illinois Supreme Court rejected Stanley’s Equal Protection claim, holding that his actual fitness as a parent was irrelevant because he and the children’s mother were unmarried.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>50623</td>\n",
       "      <td>Giglio v. United States</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-29</td>\n",
       "      <td>John Giglio</td>\n",
       "      <td>United States</td>\n",
       "      <td>Giglio</td>\n",
       "      <td>John Giglio was convicted of passing forged money orders. While his appeal to the you.S. Court of Appeals for the Second Circuit was pending, Giglio’s counsel discovered new evidence. The evidence indicated that the prosecution failed to disclose that it promised a key witness immunity from prosecution in exchange for testimony against Giglio. The district court denied Giglio’s motion for a new trial, finding that the error did not affect the verdict. The Court of Appeals affirmed.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index     ID                     name  \\\n",
       "0      0  50606              Roe v. Wade   \n",
       "1      1  50613      Stanley v. Illinois   \n",
       "2      2  50623  Giglio v. United States   \n",
       "\n",
       "                                      href          first_party  \\\n",
       "0    https://api.oyez.org/cases/1971/70-18             Jane Roe   \n",
       "1  https://api.oyez.org/cases/1971/70-5014  Peter Stanley, Sr.    \n",
       "2    https://api.oyez.org/cases/1971/70-29         John Giglio    \n",
       "\n",
       "    second_party winning_party  \\\n",
       "0     Henry Wade      Jane Roe   \n",
       "1       Illinois       Stanley   \n",
       "2  United States        Giglio   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      facts  \n",
       "0                                                                                                                                                                                                                                                             In 1970, Jane Roe (a fictional name used in court documents to protect the plaintiff’s identity) filed a lawsuit against Henry Wade, the district attorney of Dallas County, Texas, where she resided, challenging a Texas law making abortion illegal except by a doctor’s orders to save a woman’s life. In her lawsuit, Roe alleged that the state laws were unconstitutionally vague and abridged her right of personal privacy, protected by the First, Fourth, Fifth, Ninth, and Fourteenth Amendments.  \n",
       "1  Joan Stanley had three children with Peter Stanley. The Stanleys never married, but lived together off and on for 18 years. When Joan died, the State of Illinois took the children. Under Illinois law, unwed fathers were presumed unfit parents regardless of their actual fitness and their children became wards of the state. Peter appealed the decision, arguing that the Illinois law violated the Equal Protection Clause of the Fourteenth Amendment because unwed mothers were not deprived of their children without a showing that they were actually unfit parents. The Illinois Supreme Court rejected Stanley’s Equal Protection claim, holding that his actual fitness as a parent was irrelevant because he and the children’s mother were unmarried.  \n",
       "2                                                                                                                                                                                                                                                                    John Giglio was convicted of passing forged money orders. While his appeal to the you.S. Court of Appeals for the Second Circuit was pending, Giglio’s counsel discovered new evidence. The evidence indicated that the prosecution failed to disclose that it promised a key witness immunity from prosecution in exchange for testimony against Giglio. The district court denied Giglio’s motion for a new trial, finding that the error did not affect the verdict. The Court of Appeals affirmed.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(df.head(n=3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41f4afca",
   "metadata": {},
   "source": [
    "# 2. Export dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d3993bd",
   "metadata": {},
   "source": [
    "## Determine winner_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0e89cfd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def levenshtein_distance(s1, s2):\n",
    "    if len(s1) > len(s2):\n",
    "        s1, s2 = s2, s1\n",
    "\n",
    "    distances = range(len(s1) + 1)\n",
    "    for i2, c2 in enumerate(s2):\n",
    "        distances_ = [i2+1]\n",
    "        for i1, c1 in enumerate(s1):\n",
    "            if c1 == c2:\n",
    "                distances_.append(distances[i1])\n",
    "            else:\n",
    "                distances_.append(1 + min((distances[i1], distances[i1 + 1], distances_[-1])))\n",
    "        distances = distances_\n",
    "    return distances[-1]\n",
    "\n",
    "def get_abbreviation(word):\n",
    "    return ''.join(w[0] for w in word.split() if w[0].isupper())\n",
    "\n",
    "def compute_winner(row):\n",
    "    first_party = row['first_party']\n",
    "    second_party = row['second_party']\n",
    "    winning_party = row['winning_party']\n",
    "    winner = difflib.get_close_matches(winning_party, [first_party, second_party])\n",
    "    if len(winner) == 0:\n",
    "        if first_party in winning_party or winning_party in first_party:\n",
    "            return 0\n",
    "        elif second_party in winning_party or winning_party in second_party:\n",
    "            return 1\n",
    "        else:\n",
    "            if winning_party.isupper():\n",
    "                if winning_party == get_abbreviation(first_party):\n",
    "                    return 0\n",
    "                elif winning_party == get_abbreviation(second_party):\n",
    "                    return 1\n",
    "            d1 = levenshtein_distance(first_party, winning_party)\n",
    "            d2 = levenshtein_distance(second_party, winning_party)\n",
    "            if d1 <= d2:\n",
    "                return 0\n",
    "            else:\n",
    "                return 1\n",
    "    else:\n",
    "        winner = winner[0]\n",
    "        if winner == first_party:\n",
    "            return 0\n",
    "        elif winner == second_party:\n",
    "            return 1\n",
    "        else:\n",
    "            return -1\n",
    "\n",
    "df['winner_index'] = df.apply(lambda row: compute_winner(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "21611c69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>ID</th>\n",
       "      <th>name</th>\n",
       "      <th>href</th>\n",
       "      <th>first_party</th>\n",
       "      <th>second_party</th>\n",
       "      <th>winning_party</th>\n",
       "      <th>facts</th>\n",
       "      <th>winner_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>50606</td>\n",
       "      <td>Roe v. Wade</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-18</td>\n",
       "      <td>Jane Roe</td>\n",
       "      <td>Henry Wade</td>\n",
       "      <td>Jane Roe</td>\n",
       "      <td>In 1970, Jane Roe (a fictional name used in court documents to protect the plaintiff’s identity) filed a lawsuit against Henry Wade, the district attorney of Dallas County, Texas, where she resided, challenging a Texas law making abortion illegal except by a doctor’s orders to save a woman’s life. In her lawsuit, Roe alleged that the state laws were unconstitutionally vague and abridged her right of personal privacy, protected by the First, Fourth, Fifth, Ninth, and Fourteenth Amendments.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>50613</td>\n",
       "      <td>Stanley v. Illinois</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-5014</td>\n",
       "      <td>Peter Stanley, Sr.</td>\n",
       "      <td>Illinois</td>\n",
       "      <td>Stanley</td>\n",
       "      <td>Joan Stanley had three children with Peter Stanley. The Stanleys never married, but lived together off and on for 18 years. When Joan died, the State of Illinois took the children. Under Illinois law, unwed fathers were presumed unfit parents regardless of their actual fitness and their children became wards of the state. Peter appealed the decision, arguing that the Illinois law violated the Equal Protection Clause of the Fourteenth Amendment because unwed mothers were not deprived of their children without a showing that they were actually unfit parents. The Illinois Supreme Court rejected Stanley’s Equal Protection claim, holding that his actual fitness as a parent was irrelevant because he and the children’s mother were unmarried.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>50623</td>\n",
       "      <td>Giglio v. United States</td>\n",
       "      <td>https://api.oyez.org/cases/1971/70-29</td>\n",
       "      <td>John Giglio</td>\n",
       "      <td>United States</td>\n",
       "      <td>Giglio</td>\n",
       "      <td>John Giglio was convicted of passing forged money orders. While his appeal to the you.S. Court of Appeals for the Second Circuit was pending, Giglio’s counsel discovered new evidence. The evidence indicated that the prosecution failed to disclose that it promised a key witness immunity from prosecution in exchange for testimony against Giglio. The district court denied Giglio’s motion for a new trial, finding that the error did not affect the verdict. The Court of Appeals affirmed.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index     ID                     name  \\\n",
       "0      0  50606              Roe v. Wade   \n",
       "1      1  50613      Stanley v. Illinois   \n",
       "2      2  50623  Giglio v. United States   \n",
       "\n",
       "                                      href          first_party  \\\n",
       "0    https://api.oyez.org/cases/1971/70-18             Jane Roe   \n",
       "1  https://api.oyez.org/cases/1971/70-5014  Peter Stanley, Sr.    \n",
       "2    https://api.oyez.org/cases/1971/70-29         John Giglio    \n",
       "\n",
       "    second_party winning_party  \\\n",
       "0     Henry Wade      Jane Roe   \n",
       "1       Illinois       Stanley   \n",
       "2  United States        Giglio   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      facts  \\\n",
       "0                                                                                                                                                                                                                                                             In 1970, Jane Roe (a fictional name used in court documents to protect the plaintiff’s identity) filed a lawsuit against Henry Wade, the district attorney of Dallas County, Texas, where she resided, challenging a Texas law making abortion illegal except by a doctor’s orders to save a woman’s life. In her lawsuit, Roe alleged that the state laws were unconstitutionally vague and abridged her right of personal privacy, protected by the First, Fourth, Fifth, Ninth, and Fourteenth Amendments.   \n",
       "1  Joan Stanley had three children with Peter Stanley. The Stanleys never married, but lived together off and on for 18 years. When Joan died, the State of Illinois took the children. Under Illinois law, unwed fathers were presumed unfit parents regardless of their actual fitness and their children became wards of the state. Peter appealed the decision, arguing that the Illinois law violated the Equal Protection Clause of the Fourteenth Amendment because unwed mothers were not deprived of their children without a showing that they were actually unfit parents. The Illinois Supreme Court rejected Stanley’s Equal Protection claim, holding that his actual fitness as a parent was irrelevant because he and the children’s mother were unmarried.   \n",
       "2                                                                                                                                                                                                                                                                    John Giglio was convicted of passing forged money orders. While his appeal to the you.S. Court of Appeals for the Second Circuit was pending, Giglio’s counsel discovered new evidence. The evidence indicated that the prosecution failed to disclose that it promised a key witness immunity from prosecution in exchange for testimony against Giglio. The district court denied Giglio’s motion for a new trial, finding that the error did not affect the verdict. The Court of Appeals affirmed.   \n",
       "\n",
       "   winner_index  \n",
       "0             0  \n",
       "1             0  \n",
       "2             0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(df.head(n=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "85236f9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1]\n",
      "[2114  270]\n"
     ]
    }
   ],
   "source": [
    "# Show the data inbalance\n",
    "index, counts = np.unique(df['winner_index'].values,return_counts=True)\n",
    "print(index)\n",
    "print(counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a88f4e0",
   "metadata": {},
   "source": [
    "## Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "23276855",
   "metadata": {},
   "outputs": [],
   "source": [
    "# export each class into its own file\n",
    "df[df['winner_index']==0].to_pickle('class0.pkl')\n",
    "df[df['winner_index']==1].to_pickle('class1.pkl')\n",
    "\n",
    "df.to_pickle('preprocessed_dataset.pkl')\n",
    "\n",
    "# test importing\n",
    "# df0 = pd.read_pickle('class0.pkl')\n",
    "# df1 = pd.read_pickle('class1.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}