diff --git a/Solutions.ipynb b/Solutions.ipynb new file mode 100644 index 0000000..c4bdd63 --- /dev/null +++ b/Solutions.ipynb @@ -0,0 +1,876 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "621932c3", + "metadata": {}, + "source": [ + "# Handling data imbalance (classification)" + ] + }, + { + "cell_type": "markdown", + "id": "4048dbbc", + "metadata": {}, + "source": [ + "- Import the required libraries and modules that you would need." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "f1aae897", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.neural_network import MLPRegressor\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import StandardScaler\n", + "from sklearn.preprocessing import OneHotEncoder, Normalizer, LabelEncoder\n", + "from sklearn import metrics\n", + "from imblearn.over_sampling import SMOTE\n", + "from imblearn.under_sampling import TomekLinks\n", + "import warnings\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "markdown", + "id": "429fbf9d", + "metadata": {}, + "source": [ + "- Read that data into Python and call the dataframe churnData." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ab7ad729", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
genderSeniorCitizenPartnerDependentstenurePhoneServiceOnlineSecurityOnlineBackupDeviceProtectionTechSupportStreamingTVStreamingMoviesContractMonthlyChargesTotalChargesChurn
0Female0YesNo1NoNoYesNoNoNoNoMonth-to-month29.8529.85No
1Male0NoNo34YesYesNoYesNoNoNoOne year56.951889.5No
2Male0NoNo2YesYesYesNoNoNoNoMonth-to-month53.85108.15Yes
3Male0NoNo45NoYesNoYesYesNoNoOne year42.301840.75No
4Female0NoNo2YesNoNoNoNoNoNoMonth-to-month70.70151.65Yes
\n", + "
" + ], + "text/plain": [ + " gender SeniorCitizen Partner Dependents tenure PhoneService \\\n", + "0 Female 0 Yes No 1 No \n", + "1 Male 0 No No 34 Yes \n", + "2 Male 0 No No 2 Yes \n", + "3 Male 0 No No 45 No \n", + "4 Female 0 No No 2 Yes \n", + "\n", + " OnlineSecurity OnlineBackup DeviceProtection TechSupport StreamingTV \\\n", + "0 No Yes No No No \n", + "1 Yes No Yes No No \n", + "2 Yes Yes No No No \n", + "3 Yes No Yes Yes No \n", + "4 No No No No No \n", + "\n", + " StreamingMovies Contract MonthlyCharges TotalCharges Churn \n", + "0 No Month-to-month 29.85 29.85 No \n", + "1 No One year 56.95 1889.5 No \n", + "2 No Month-to-month 53.85 108.15 Yes \n", + "3 No One year 42.30 1840.75 No \n", + "4 No Month-to-month 70.70 151.65 Yes " + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "churnData = pd.read_csv(\"./files_for_lab/Customer-Churn.csv\")\n", + "churnData.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0cc41cf6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(7043, 16)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "churnData.shape" + ] + }, + { + "cell_type": "markdown", + "id": "8ff91084", + "metadata": {}, + "source": [ + "- Check the datatypes of all the columns in the data. You would see that the column TotalCharges is object type. Convert this column into numeric type using pd.to_numeric function." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a93162d9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "gender object\n", + "SeniorCitizen int64\n", + "Partner object\n", + "Dependents object\n", + "tenure int64\n", + "PhoneService object\n", + "OnlineSecurity object\n", + "OnlineBackup object\n", + "DeviceProtection object\n", + "TechSupport object\n", + "StreamingTV object\n", + "StreamingMovies object\n", + "Contract object\n", + "MonthlyCharges float64\n", + "TotalCharges object\n", + "Churn object\n", + "dtype: object" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "churnData.dtypes" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "389a5647", + "metadata": {}, + "outputs": [], + "source": [ + "churnData[\"TotalCharges\"] = pd.to_numeric(churnData[\"TotalCharges\"], errors = 'coerce')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "c0fb4498", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dtype('float64')" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "churnData[\"TotalCharges\"].dtypes" + ] + }, + { + "cell_type": "markdown", + "id": "53fe6978", + "metadata": {}, + "source": [ + "- Check for null values in the dataframe. Replace the null values." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "d1b4e62b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "gender 0\n", + "SeniorCitizen 0\n", + "Partner 0\n", + "Dependents 0\n", + "tenure 0\n", + "PhoneService 0\n", + "OnlineSecurity 0\n", + "OnlineBackup 0\n", + "DeviceProtection 0\n", + "TechSupport 0\n", + "StreamingTV 0\n", + "StreamingMovies 0\n", + "Contract 0\n", + "MonthlyCharges 0\n", + "TotalCharges 11\n", + "Churn 0\n", + "dtype: int64" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "churnData.isnull().sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "118b5fc3", + "metadata": {}, + "outputs": [], + "source": [ + "churnData['TotalCharges'].fillna((churnData['TotalCharges'].mean()), inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "fd09115d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "gender 0\n", + "SeniorCitizen 0\n", + "Partner 0\n", + "Dependents 0\n", + "tenure 0\n", + "PhoneService 0\n", + "OnlineSecurity 0\n", + "OnlineBackup 0\n", + "DeviceProtection 0\n", + "TechSupport 0\n", + "StreamingTV 0\n", + "StreamingMovies 0\n", + "Contract 0\n", + "MonthlyCharges 0\n", + "TotalCharges 0\n", + "Churn 0\n", + "dtype: int64" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "churnData.isnull().sum()" + ] + }, + { + "cell_type": "markdown", + "id": "e95fd27c", + "metadata": {}, + "source": [ + "- Scale the features (tenure, SeniorCitizen, MonthlyCharges and TotalCharges) either by using normalizer or a standard scaler." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "be1dd041", + "metadata": {}, + "outputs": [], + "source": [ + "y = churnData['Churn']" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "759e1108", + "metadata": {}, + "outputs": [], + "source": [ + "X = churnData[['tenure', 'SeniorCitizen', 'MonthlyCharges', 'TotalCharges']]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "ccf69688", + "metadata": {}, + "outputs": [], + "source": [ + "transformer = Normalizer() \n", + "transformer.fit(X)\n", + "x_normalized = transformer.transform(X)" + ] + }, + { + "cell_type": "markdown", + "id": "8995267e", + "metadata": {}, + "source": [ + "- Split the data into a training set and a test set." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "20f7eedd", + "metadata": {}, + "outputs": [], + "source": [ + "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)" + ] + }, + { + "cell_type": "markdown", + "id": "9146738d", + "metadata": {}, + "source": [ + "- Fit a logistic regression model on the training data." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "e8f5bdc4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LogisticRegression()" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logreg = LogisticRegression()\n", + "logreg.fit(X_train, y_train)" + ] + }, + { + "cell_type": "markdown", + "id": "348cfc64", + "metadata": {}, + "source": [ + "- Check the accuracy on the test data." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "953e4c7a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.7825099375354913" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logreg.score(X_test, y_test)" + ] + }, + { + "cell_type": "markdown", + "id": "50261417", + "metadata": {}, + "source": [ + "### Imbalance management" + ] + }, + { + "cell_type": "markdown", + "id": "6f170e41", + "metadata": {}, + "source": [ + "- Check for the imbalance." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "b8bbf6e6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "No 5174\n", + "Yes 1869\n", + "Name: Churn, dtype: int64" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "y.value_counts()" + ] + }, + { + "cell_type": "markdown", + "id": "31e8a35a", + "metadata": {}, + "source": [ + "- Use upsampling and downsampling to create a balance between the two classes. Each time fit the model and see how the accuracy of the model is." + ] + }, + { + "cell_type": "markdown", + "id": "38211768", + "metadata": {}, + "source": [ + "#### Upsample l first method" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "17b7915f", + "metadata": {}, + "outputs": [], + "source": [ + "data = pd.concat([X_train, y_train], axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "3c50c437", + "metadata": {}, + "outputs": [], + "source": [ + "churn_no = data[data['Churn'] == 'No'].sample(5174, replace=True)\n", + "churn_yes = data[data['Churn'] == 'Yes'].sample(5174, replace=True) " + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "563bb703", + "metadata": {}, + "outputs": [], + "source": [ + "churn_no = churn_no.sample(len(churn_no), replace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "16d3fa3c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Yes 5174\n", + "No 5174\n", + "Name: Churn, dtype: int64" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = pd.concat([churn_no, churn_yes], axis=0)\n", + "data = data.sample(frac=1)\n", + "data['Churn'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "5c9eeb3b", + "metadata": {}, + "outputs": [], + "source": [ + "upsampled_train = pd.concat([churn_yes, churn_no]).sample(frac=1)\n", + "other = upsampled_train.drop(\"Churn\", axis = 1)\n", + "target = upsampled_train[\"Churn\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "385f8240", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.6842703009653606" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logreg.fit(other, target)\n", + "logreg.score(X_test, y_test)" + ] + }, + { + "cell_type": "markdown", + "id": "41e7737a", + "metadata": {}, + "source": [ + "#### Upsample l second method: SMOTE" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "41e14656", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Yes 5174\n", + "No 5174\n", + "Name: Churn, dtype: int64" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "smote = SMOTE()\n", + "\n", + "X_sm, y_sm = smote.fit_resample(X, y)\n", + "y_sm.value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "c9dbec39", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.7137989778534923" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logreg.fit(X_sm, y_sm)\n", + "logreg.score(X_test, y_test)" + ] + }, + { + "cell_type": "markdown", + "id": "d56e9956", + "metadata": {}, + "source": [ + "#### Downsample l first method" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "6d843681", + "metadata": {}, + "outputs": [], + "source": [ + "data1 = pd.concat([X_train, y_train], axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "4d1f94bf", + "metadata": {}, + "outputs": [], + "source": [ + "No = data1[data1['Churn'] == 'No'].sample(5174, replace=True)\n", + "Yes = data1[data1['Churn'] == 'Yes'].sample(5174, replace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "d8819efe", + "metadata": {}, + "outputs": [], + "source": [ + "downsampled = pd.concat([Yes, No]).sample(frac=1)\n", + "X_dw_train = downsampled.drop(\"Churn\", axis = 1)\n", + "y_dw_train = downsampled[\"Churn\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "b4762093", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.7030096536059057" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logreg.fit(X_dw_train, y_dw_train)\n", + "logreg.score(X_test, y_test)" + ] + }, + { + "cell_type": "markdown", + "id": "f95f8d3a", + "metadata": {}, + "source": [ + "#### Downsample l second method: Tomek Links " + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "4f431e3d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "No 4620\n", + "Yes 1869\n", + "Name: Churn, dtype: int64" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tomek = TomekLinks()\n", + "X_tl, y_tl = tomek.fit_resample(X, y)\n", + "y_tl.value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "14d5175a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.7649063032367973" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "logreg.fit(X_tl, y_tl)\n", + "logreg.score(X_test, y_test)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}