unit-project/3/notebooks/unit-project-3-starter-code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DS-SF-30 | Unit Project 3: Machine Learning Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this project, you will perform a logistic regression on the admissions data we've been working with in Unit Projects 1 and 2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 10)\n",
    "pd.set_option('display.max_columns', 10)\n",
    "pd.set_option('display.notebook_repr_html', True)\n",
    "\n",
    "import statsmodels.formula.api as smf\n",
    "\n",
    "from sklearn import linear_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>admit</th>\n",
       "      <th>gre</th>\n",
       "      <th>gpa</th>\n",
       "      <th>prestige</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>380.0</td>\n",
       "      <td>3.61</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>660.0</td>\n",
       "      <td>3.67</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>800.0</td>\n",
       "      <td>4.00</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>640.0</td>\n",
       "      <td>3.19</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>520.0</td>\n",
       "      <td>2.93</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>0</td>\n",
       "      <td>620.0</td>\n",
       "      <td>4.00</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>0</td>\n",
       "      <td>560.0</td>\n",
       "      <td>3.04</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>0</td>\n",
       "      <td>460.0</td>\n",
       "      <td>2.63</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>0</td>\n",
       "      <td>700.0</td>\n",
       "      <td>3.65</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>0</td>\n",
       "      <td>600.0</td>\n",
       "      <td>3.89</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>397 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     admit    gre   gpa  prestige\n",
       "0        0  380.0  3.61       3.0\n",
       "1        1  660.0  3.67       3.0\n",
       "2        1  800.0  4.00       1.0\n",
       "3        1  640.0  3.19       4.0\n",
       "4        0  520.0  2.93       4.0\n",
       "..     ...    ...   ...       ...\n",
       "395      0  620.0  4.00       2.0\n",
       "396      0  560.0  3.04       3.0\n",
       "397      0  460.0  2.63       2.0\n",
       "398      0  700.0  3.65       2.0\n",
       "399      0  600.0  3.89       3.0\n",
       "\n",
       "[397 rows x 4 columns]"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(os.path.join('..', '..', 'dataset', 'dataset-ucla-admissions.csv'))\n",
    "df.dropna(inplace = True)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part A.  Frequency Table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 1.  Create a frequency table for `prestige` and whether an applicant was admitted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0    148\n",
       "3.0    121\n",
       "4.0     67\n",
       "1.0     61\n",
       "Name: prestige, dtype: int64"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prestige = df.prestige\n",
    "df.prestige.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part B.  Variable Transformations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 2.  Create a one-hot encoding for `prestige`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prestige_1.0</th>\n",
       "      <th>prestige_2.0</th>\n",
       "      <th>prestige_3.0</th>\n",
       "      <th>prestige_4.0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>397 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     prestige_1.0  prestige_2.0  prestige_3.0  prestige_4.0\n",
       "0               0             0             1             0\n",
       "1               0             0             1             0\n",
       "2               1             0             0             0\n",
       "3               0             0             0             1\n",
       "4               0             0             0             1\n",
       "..            ...           ...           ...           ...\n",
       "395             0             1             0             0\n",
       "396             0             0             1             0\n",
       "397             0             1             0             0\n",
       "398             0             1             0             0\n",
       "399             0             0             1             0\n",
       "\n",
       "[397 rows x 4 columns]"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_prestige_onehot = pd.get_dummies(df.prestige, prefix=\"prestige\")\n",
    "df_prestige_onehot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 3.  How many of these binary variables do we need for modeling?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Answer: n-1 will capture all the information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 4.  Why are we doing this?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Answer: one hot encoding transforms categorial features into a format more easily consumed for classification models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 5.  Add all these binary variables in the dataset and remove the now redundant `prestige` feature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.drop('prestige', axis=1, inplace=True)\n",
    "df = df.join(df_prestige_onehot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part C.  Hand calculating odds ratios"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's develop our intuition about expected outcomes by hand calculating odds ratios."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 6.  Create a frequency table for `prestige = 1` and whether an applicant was admitted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>prestige_1.0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>admit</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>243</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>93</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "prestige_1.0    0   1\n",
       "admit                \n",
       "0             243  28\n",
       "1              93  33"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(df['admit'], df['prestige_1.0'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>prestige_1.0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>admit</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.896679</td>\n",
       "      <td>0.103321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.738095</td>\n",
       "      <td>0.261905</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "prestige_1.0         0         1\n",
       "admit                           \n",
       "0             0.896679  0.103321\n",
       "1             0.738095  0.261905"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(df['admit'], df['prestige_1.0'], normalize='index')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 7.  Use the frequency table above to calculate the odds of being admitted to graduate school for applicants that attended the most prestigious undergraduate schools."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.1785714285714288"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prob = 33. / (28 + 33)\n",
    "oddsA = prob / (1 - prob)\n",
    "oddsA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 8.  Now calculate the odds of admission for undergraduates who did not attend a #1 ranked college."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3827160493827161"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prob = 93. / (243 + 93)\n",
    "oddsB = prob / (1 - prob)\n",
    "oddsB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 9.  Finally, what's the odds ratio?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.079493087557604"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oddsA / oddsB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 10.  Write this finding in a sentence."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Answer: 3.07 times more likely to be admitted if you went to a prestigious college"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 11.  Use the frequency table above to calculate the odds of being admitted to graduate school for applicants that attended the least prestigious undergraduate schools.  Then calculate their odds ratio of being admitted to UCLA.  Finally, write this finding in a sentence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>prestige_4.0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>admit</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>216</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>114</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "prestige_4.0    0   1\n",
       "admit                \n",
       "0             216  55\n",
       "1             114  12"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(df['admit'], df['prestige_4.0'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4133971291866028"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pA = 12. / (12 + 55)\n",
    "oddsA = pA / ( 1 - pA )\n",
    "\n",
    "pB = 114. / (114 + 216)\n",
    "oddsB = pB / (1 - pB)\n",
    "\n",
    "oddsA / oddsB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Answer: you are 0.41 as likely to get in if you went to a low prestige school"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part C. Analysis using `statsmodels`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 12.  Fit a logistic regression model predicting admission into UCLA using `gre`, `gpa`, and the `prestige` of the undergraduate schools.  Use the highest prestige undergraduate schools as your reference point."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# finally drop the unnecessary prestige column\n",
    "df.drop('prestige_1.0', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# manually add intercept\n",
    "df['intercept'] = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_cols = df.columns[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.573854\n",
      "         Iterations 6\n"
     ]
    }
   ],
   "source": [
    "# from patsy import dmatrices\n",
    "# y, X = dmatrices('admit ~ gre + gpa + C(prestige)', df, return_type = 'dataframe')\n",
    "\n",
    "import statsmodels.discrete.discrete_model as sm\n",
    "\n",
    "y = df['admit']\n",
    "X = df[train_cols]\n",
    "\n",
    "logit = sm.Logit(y, X)\n",
    "result = logit.fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 13.  Print the model's summary results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "gre             0.002218\n",
       "gpa             0.779337\n",
       "prestige_2.0   -0.680137\n",
       "prestige_3.0   -1.338677\n",
       "prestige_4.0   -1.553411\n",
       "intercept      -3.876854\n",
       "dtype: float64"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>Logit Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>       <td>admit</td>      <th>  No. Observations:  </th>  <td>   397</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>               <td>Logit</td>      <th>  Df Residuals:      </th>  <td>   391</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>               <td>MLE</td>       <th>  Df Model:          </th>  <td>     5</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>          <td>Mon, 06 Feb 2017</td> <th>  Pseudo R-squ.:     </th>  <td>0.08166</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>              <td>23:15:51</td>     <th>  Log-Likelihood:    </th> <td> -227.82</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>converged:</th>           <td>True</td>       <th>  LL-Null:           </th> <td> -248.08</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th> </th>                      <td> </td>        <th>  LLR p-value:       </th> <td>1.176e-07</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "        <td></td>          <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th> <th>[95.0% Conf. Int.]</th> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gre</th>          <td>    0.0022</td> <td>    0.001</td> <td>    2.028</td> <td> 0.043</td> <td> 7.44e-05     0.004</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>gpa</th>          <td>    0.7793</td> <td>    0.333</td> <td>    2.344</td> <td> 0.019</td> <td>    0.128     1.431</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prestige_2.0</th> <td>   -0.6801</td> <td>    0.317</td> <td>   -2.146</td> <td> 0.032</td> <td>   -1.301    -0.059</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prestige_3.0</th> <td>   -1.3387</td> <td>    0.345</td> <td>   -3.882</td> <td> 0.000</td> <td>   -2.015    -0.663</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>prestige_4.0</th> <td>   -1.5534</td> <td>    0.417</td> <td>   -3.721</td> <td> 0.000</td> <td>   -2.372    -0.735</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>intercept</th>    <td>   -3.8769</td> <td>    1.142</td> <td>   -3.393</td> <td> 0.001</td> <td>   -6.116    -1.638</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                           Logit Regression Results                           \n",
       "==============================================================================\n",
       "Dep. Variable:                  admit   No. Observations:                  397\n",
       "Model:                          Logit   Df Residuals:                      391\n",
       "Method:                           MLE   Df Model:                            5\n",
       "Date:                Mon, 06 Feb 2017   Pseudo R-squ.:                 0.08166\n",
       "Time:                        23:15:51   Log-Likelihood:                -227.82\n",
       "converged:                       True   LL-Null:                       -248.08\n",
       "                                        LLR p-value:                 1.176e-07\n",
       "================================================================================\n",
       "                   coef    std err          z      P>|z|      [95.0% Conf. Int.]\n",
       "--------------------------------------------------------------------------------\n",
       "gre              0.0022      0.001      2.028      0.043      7.44e-05     0.004\n",
       "gpa              0.7793      0.333      2.344      0.019         0.128     1.431\n",
       "prestige_2.0    -0.6801      0.317     -2.146      0.032        -1.301    -0.059\n",
       "prestige_3.0    -1.3387      0.345     -3.882      0.000        -2.015    -0.663\n",
       "prestige_4.0    -1.5534      0.417     -3.721      0.000        -2.372    -0.735\n",
       "intercept       -3.8769      1.142     -3.393      0.001        -6.116    -1.638\n",
       "================================================================================\n",
       "\"\"\""
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 14.  What are the odds ratios of the different features and their 95% confidence intervals?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.573854\n",
      "         Iterations 6\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gre</th>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.004362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpa</th>\n",
       "      <td>0.127619</td>\n",
       "      <td>1.431056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prestige_2.0</th>\n",
       "      <td>-1.301337</td>\n",
       "      <td>-0.058936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prestige_3.0</th>\n",
       "      <td>-2.014579</td>\n",
       "      <td>-0.662776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prestige_4.0</th>\n",
       "      <td>-2.371624</td>\n",
       "      <td>-0.735197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>intercept</th>\n",
       "      <td>-6.116077</td>\n",
       "      <td>-1.637631</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     0         1\n",
       "gre           0.000074  0.004362\n",
       "gpa           0.127619  1.431056\n",
       "prestige_2.0 -1.301337 -0.058936\n",
       "prestige_3.0 -2.014579 -0.662776\n",
       "prestige_4.0 -2.371624 -0.735197\n",
       "intercept    -6.116077 -1.637631"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logit.fit().conf_int(alpha=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gre             1.002221\n",
      "gpa             2.180027\n",
      "prestige_2.0    0.506548\n",
      "prestige_3.0    0.262192\n",
      "prestige_4.0    0.211525\n",
      "intercept       0.020716\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# odds ratios\n",
    "print np.exp(result.params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 15.  Interpret the odds ratio for `prestige = 2`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Answer: Expect odds to decrease by about 50% if prestige = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 16.  Interpret the odds ratio of `gpa`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Answer: Each point in GPA increases chance of admittance by factor of 2.18"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 17.  Assuming a student with a GRE of 800 and a GPA of 4.  What is his/her probability of admission  if he/she come from a tier-1, tier-2, tier-3, or tier-4 undergraduate school?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gpa</th>\n",
       "      <th>gre</th>\n",
       "      <th>intercept</th>\n",
       "      <th>prestige_1.0</th>\n",
       "      <th>prestige_2.0</th>\n",
       "      <th>prestige_3.0</th>\n",
       "      <th>prestige_4.0</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>800</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.734040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>800</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.582995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>800</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.419833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>800</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.368608</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   gpa  gre  intercept  prestige_1.0  prestige_2.0  prestige_3.0  \\\n",
       "0    4  800        1.0             1             0             0   \n",
       "1    4  800        1.0             0             1             0   \n",
       "2    4  800        1.0             0             0             1   \n",
       "3    4  800        1.0             0             0             0   \n",
       "\n",
       "   prestige_4.0  prediction  \n",
       "0             0    0.734040  \n",
       "1             0    0.582995  \n",
       "2             0    0.419833  \n",
       "3             1    0.368608  "
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "guess = pd.DataFrame({\n",
    "        'gre': [800,800,800,800],\n",
    "        'gpa': [4,4,4,4],\n",
    "        'prestige': [1., 2., 3., 4.],\n",
    "        'intercept': [1., 1., 1., 1.]\n",
    "    })\n",
    "\n",
    "dummy_pres = pd.get_dummies(guess['prestige'], prefix='prestige')\n",
    "\n",
    "guess = guess.join(dummy_pres)\n",
    "guess.drop('prestige', axis=1, inplace=True)\n",
    "\n",
    "prediction = result.predict(guess[train_cols])\n",
    "\n",
    "with_pred = guess.copy()\n",
    "with_pred['prediction'] = prediction \n",
    "\n",
    "with_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Answer:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part D. Moving the model from `statsmodels` to `sklearn`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 18.  Let's assume we are satisfied with our model.  Remodel it (same features) using `sklearn`.  When creating the logistic regression model with `LogisticRegression(C = 10 ** 2)`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = linear_model.LogisticRegression(C = 10 ** 2)\n",
    "skresult = model.fit(X, y=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 19.  What are the odds ratios for the different variables and how do they compare with the odds ratios calculated with `statsmodels`?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.0020999 ,  2.05051732,  0.48210089,  0.24652421,  0.20131997,\n",
       "         0.16981759]])"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.exp(skresult.coef_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Answer: They are different, presumably because of the different C value, but I probably did something wrong"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Question 20.  Again, assuming a student with a GRE of 800 and a GPA of 4.  What is his/her probability of admission  if he/she come from a tier-1, tier-2, tier-3, or tier-4 undergraduate school?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gpa</th>\n",
       "      <th>gre</th>\n",
       "      <th>intercept</th>\n",
       "      <th>prestige_1.0</th>\n",
       "      <th>prestige_2.0</th>\n",
       "      <th>prestige_3.0</th>\n",
       "      <th>prestige_4.0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>800</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.268065</td>\n",
       "      <td>0.731935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>800</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.431714</td>\n",
       "      <td>0.568286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>800</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.597686</td>\n",
       "      <td>0.402314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>800</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.645289</td>\n",
       "      <td>0.354711</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   gpa  gre  intercept  prestige_1.0  prestige_2.0  prestige_3.0  \\\n",
       "0    4  800        1.0             1             0             0   \n",
       "1    4  800        1.0             0             1             0   \n",
       "2    4  800        1.0             0             0             1   \n",
       "3    4  800        1.0             0             0             0   \n",
       "\n",
       "   prestige_4.0         0         1  \n",
       "0             0  0.268065  0.731935  \n",
       "1             0  0.431714  0.568286  \n",
       "2             0  0.597686  0.402314  \n",
       "3             1  0.645289  0.354711  "
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction = skresult.predict_proba(guess[train_cols])\n",
    "prediction\n",
    "\n",
    "guess.join(pd.DataFrame(prediction))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Answer: About the same predictions as prior model"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}