Predict income.py

# -*- coding: utf-8 -*-
"""Final Project.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1e7dEVgwW-WYlHBQp4iEuR5dJww_XlikF
"""

import numpy as np
import pandas as pd


features = ["Age", "Workclass", "fnlwgt", "Education", "Education-Num", "Martial Status",
        "Occupation", "Relationship", "Race", "Sex", "Capital Gain", "Capital Loss",
        "Hours per week", "Country", "Target"]
data=pd.read_csv('https://raw.githubusercontent.com/trongbao0208/Machine-Learning-Final-Project/main/adult.data', names = features, sep=r'\s*,\s*', 
                             engine='python', na_values="?")

"""## Inspection and Basic Exploratory Data Analysis"""

#Inspect the first 6 rows of the dataset
data = data.dropna()
data.head(6)

data.describe()

age = data['Age'].unique()
age.sort()
print(age)

"""## Preprocessing the dataset"""

# Convert Target Column to 1 if >50k and 0 if <= 50k
from sklearn.preprocessing import LabelEncoder

le = LabelEncoder()
data['Target_int'] = le.fit_transform(data['Target'])

from sklearn.model_selection import train_test_split
### features to use: Age, education-num, occupation, race, marital status, hours per week, workclass.
col_names = ['Age', 'Workclass', 'Education-Num', 'Martial Status', 'Occupation', 'Race', 'Sex', 'Hours per week']
X = data[col_names].values
y = data['Target_int'].values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, stratify = y)

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score, GridSearchCV


num_pipe = Pipeline([#('imp', SimpleImputer(strategy='mean')),
                   ('scale', StandardScaler())
                   ])

cat_pipe = Pipeline([#('imp', SimpleImputer(strategy='most_frequent')), 
                     ('oh', OneHotEncoder(dtype=np.uint8))
                     ])

col_transf = ColumnTransformer([ 
                              ('num', num_pipe, [0, 2, 7]),
                              ('cat', cat_pipe, [1, 3, 4, 5, 6])
                              ])

"""## Model #1: Logistic Regression"""

max_it = 500
from sklearn.metrics import f1_score
full_pipe = Pipeline([
                   ('transf', col_transf),
                   ('model', LogisticRegression(max_iter = max_it))
                   ])

n_folds = 5

params = {"model__C" : np.arange(.1, 1.1, .1)}

lr_cv = GridSearchCV(full_pipe, param_grid = params, cv = n_folds, scoring = "accuracy")
lr_cv.fit(X_train, y_train)
y_pred = lr_cv.predict(X_test)
y_probas = lr_cv.predict_proba(X_test)

acc = lr_cv.best_score_
c_best = lr_cv.best_params_['model__C']

print(f"The best accuracy score over {n_folds} folds:\n {acc:.3f}")
print(f"The best value of the C parameter for this model: {c_best}.")

from sklearn.metrics import confusion_matrix, plot_confusion_matrix, classification_report

v_int_lr = lr_cv.best_estimator_['model'].intercept_
weights_lr = lr_cv.best_estimator_['model'].coef_

print(f"The weights are \n{weights_lr}\n and the bias term is {v_int_lr}.")

import matplotlib.pyplot as plt
class_names=['Less than 50k', 'More than 50k']
fig, ax = plt.subplots(figsize=(6, 6))
plot_confusion_matrix(lr_cv, X_test, y_test,
                      display_labels= class_names, 
                      cmap=plt.cm.Reds, ax=ax, values_format = 'd')
plt.title("Model 1: Logistic Regression", fontsize = 15)
plt.show()

print(classification_report(y_test, y_pred))

"""## Model #2: Random Forest Classifier"""

from sklearn.ensemble import RandomForestClassifier

rf_pipe = Pipeline([
                   ('transf', col_transf),
                   ('rf', RandomForestClassifier(n_estimators = 150, random_state = 42))
                   ])

params_rf = {'rf__min_samples_leaf': np.arange(1, 11, 1)}

rf_cv = GridSearchCV(rf_pipe, param_grid = params_rf, cv = n_folds, scoring = "accuracy")
rf_cv.fit(X_train, y_train)
y_pred_rf = rf_cv.predict(X_test)
y_probas_rf = rf_cv.predict_proba(X_test)

acc_rf = rf_cv.best_score_
best_leaf = rf_cv.best_params_['rf__min_samples_leaf']

print(f"The best accuracy score over {n_folds} folds:\n {acc_rf:.3f}")
print(f"The best number of the minimum leafs for this model: {best_leaf}.")

class_names=['Less than 50k', 'More than 50k']
fig, ax = plt.subplots(figsize=(6, 6))
plot_confusion_matrix(rf_cv, X_test, y_test,
                      display_labels= class_names, 
                      cmap=plt.cm.Blues, ax=ax, values_format = 'd')
plt.title("Model 2: Random Forest", fontsize = 15)
plt.show()

print(classification_report(y_test, y_pred_rf))

"""##Model #3: K Nearest Neighbors"""

from sklearn.neighbors import KNeighborsClassifier

knn_pipe = Pipeline([
                   ('transf', col_transf),
                   ('knn', KNeighborsClassifier())
                   ])

params_knn = {'knn__n_neighbors': np.arange(1, 11, 1)}

knn_cv = GridSearchCV(knn_pipe, param_grid = params_knn, cv = n_folds, scoring = "accuracy")
knn_cv.fit(X_train, y_train)
y_pred_knn = knn_cv.predict(X_test)
y_probas_knn = knn_cv.predict_proba(X_test)

acc_knn = knn_cv.best_score_
best_neigh = knn_cv.best_params_['knn__n_neighbors']

print(f"The best accuracy score over {n_folds} folds:\n {acc_knn:.3f}")
print(f"The best number of neighbors for this model: {best_neigh}.")

class_names=['Less than 50k', 'More than 50k']
fig, ax = plt.subplots(figsize=(6, 6))
plot_confusion_matrix(knn_cv, X_test, y_test,
                      display_labels= class_names, 
                      cmap=plt.cm.Greens, ax=ax, values_format = 'd')
plt.title("Model 3: 10-Nearest Neighbors", fontsize = 15)
plt.show()

print(classification_report(y_test, y_pred_knn))

"""##Comparison of the 3 models"""

print(f"The best accuracy score for our Logistic Regression model:\n {acc:.3f}")

print(f"The best accuracy score for our Random Forest model:\n {acc_rf:.3f}")

print(f"The best accuracy score for our K Nearest Neighbors model:\n {acc_knn:.3f}")

print(f"The model with the best overall accuracy score is our Random Forest model with a score of:\n {acc_rf:.3f}")

fig, ax = plt.subplots()

bar = plt.bar([0, 1, 2], [acc, acc_rf, acc_knn], tick_label = ["Logistic Regression", "Random Forest", '10 Nearest Neighbors'], color = ['red', 'blue', 'green'])
plt.title("Comparison of Model Accuracy scores", fontsize = 15)
plt.ylabel("Accuracy", fontsize = 12)
ax.set(ylim = [.75, .88])
colors = {acc:'red', acc_rf:'blue', acc_knn :'green'}         
labels = list(colors.keys())
handles = [plt.Rectangle((0,0),1,1, color=colors[label]) for label in labels]
plt.legend(handles, labels)
plt.show()