From 29d27a9e71ff4a9366fb9064ecbf2802f01c264c Mon Sep 17 00:00:00 2001 From: Terminal-zkolar Date: Thu, 16 Sep 2021 15:43:57 +0200 Subject: [PATCH] done --- Solutions.ipynb | 345 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 298 insertions(+), 47 deletions(-) diff --git a/Solutions.ipynb b/Solutions.ipynb index 89aa338..2b653ca 100644 --- a/Solutions.ipynb +++ b/Solutions.ipynb @@ -2,91 +2,287 @@ "cells": [ { "cell_type": "markdown", - "id": "bf32ae3b", - "metadata": {}, "source": [ "# Lab | Revisiting Machine Learning Case Study" - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "id": "73df37d0", - "metadata": {}, "source": [ "In this lab, you will use learningSet.csv file which you already have cloned in today's activities." - ] + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 66, + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import warnings\n", + "\n", + "warnings.filterwarnings('ignore')" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 67, + "source": [ + "data = pd.read_csv('learningSet.csv')" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 68, + "source": [ + "data.shape" + ], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(95412, 481)" + ] + }, + "metadata": {}, + "execution_count": 68 + } + ], + "metadata": {} }, { "cell_type": "markdown", - "id": "5bbb5252", - "metadata": {}, "source": [ "### 1. Check for null values in all the columns" - ] + ], + "metadata": {} }, { "cell_type": "code", - "execution_count": null, - "id": "dbe69e8b", - "metadata": {}, - "outputs": [], - "source": [] + "execution_count": 70, + "source": [ + "nulls = pd.DataFrame(data.isna().sum()*100/len(data), columns=['percentage'])\n", + "nulls.sort_values('percentage', ascending = False).head(80)" + ], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
percentage
RDATE_599.990567
RAMNT_599.990567
RDATE_399.746363
RAMNT_399.746363
RDATE_499.705488
......
ADATE_1621.343227
ADATE_1419.774242
ADATE_911.785729
ADATE_1110.923154
NEXTDATE10.452564
\n", + "

80 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " percentage\n", + "RDATE_5 99.990567\n", + "RAMNT_5 99.990567\n", + "RDATE_3 99.746363\n", + "RAMNT_3 99.746363\n", + "RDATE_4 99.705488\n", + "... ...\n", + "ADATE_16 21.343227\n", + "ADATE_14 19.774242\n", + "ADATE_9 11.785729\n", + "ADATE_11 10.923154\n", + "NEXTDATE 10.452564\n", + "\n", + "[80 rows x 1 columns]" + ] + }, + "metadata": {}, + "execution_count": 70 + } + ], + "metadata": {} }, { "cell_type": "markdown", - "id": "6febccdc", - "metadata": {}, "source": [ "### 2. Exclude the following variables by looking at the definitions. Create a new empty list called drop_list. We will append this list and then drop all the columns in this list later:\n", "\n", "- `OSOURCE` - symbol definitions not provided, too many categories\n", "- `ZIP CODE` - we are including state already\n" - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": null, - "id": "3a07b75a", - "metadata": {}, + "source": [ + "drop_list = []" + ], "outputs": [], - "source": [] + "metadata": {} }, { "cell_type": "markdown", - "id": "6a2470e4", - "metadata": {}, "source": [ "### 3. Identify columns that over 85% missing values" - ] + ], + "metadata": {} }, { "cell_type": "code", - "execution_count": null, - "id": "2b6f01b6", - "metadata": {}, - "outputs": [], - "source": [] + "execution_count": 71, + "source": [ + "cols_to_remove = list(nulls[nulls['percentage'] > 85].index)\n", + "cols_to_remove" + ], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "['NUMCHLD',\n", + " 'RDATE_3',\n", + " 'RDATE_4',\n", + " 'RDATE_5',\n", + " 'RDATE_6',\n", + " 'RDATE_7',\n", + " 'RDATE_10',\n", + " 'RDATE_13',\n", + " 'RDATE_15',\n", + " 'RDATE_17',\n", + " 'RDATE_20',\n", + " 'RDATE_21',\n", + " 'RDATE_23',\n", + " 'RAMNT_3',\n", + " 'RAMNT_4',\n", + " 'RAMNT_5',\n", + " 'RAMNT_6',\n", + " 'RAMNT_7',\n", + " 'RAMNT_10',\n", + " 'RAMNT_13',\n", + " 'RAMNT_15',\n", + " 'RAMNT_17',\n", + " 'RAMNT_20',\n", + " 'RAMNT_21',\n", + " 'RAMNT_23']" + ] + }, + "metadata": {}, + "execution_count": 71 + } + ], + "metadata": {} }, { "cell_type": "markdown", - "id": "70865946", - "metadata": {}, "source": [ "### 4. Remove those columns from the dataframe" - ] + ], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "drop_list = []\n", + "\n", + "for col in data:\n", + " if nulls[nulls['percentage'] > 85]\n", + " drop_list.append(col)\n", + " \n", + "return data" + ], + "metadata": {} }, { "cell_type": "code", - "execution_count": null, - "id": "ca6c4d91", - "metadata": {}, - "outputs": [], - "source": [] + "execution_count": 75, + "source": [ + "data = data.drop(['NUMCHLD', 'RDATE_3', 'RDATE_4', 'RDATE_5', 'RDATE_6', 'RDATE_7', 'RDATE_10', 'RDATE_13', 'RDATE_15', 'RDATE_17', 'RDATE_20', 'RDATE_21', 'RDATE_23', 'RAMNT_3', 'RAMNT_4', 'RAMNT_5', 'RAMNT_6', 'RAMNT_7', 'RAMNT_10', 'RAMNT_13', 'RAMNT_15', 'RAMNT_17', 'RAMNT_20', 'RAMNT_21', 'RAMNT_23'], axis=1)\n", + "data.shape" + ], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(95412, 456)" + ] + }, + "metadata": {}, + "execution_count": 75 + } + ], + "metadata": {} }, { "cell_type": "markdown", - "id": "7bbcf161", - "metadata": {}, "source": [ "### 5. Reduce the number of categories in the column GENDER. The column should only have either \"M\" for males, \"F\" for females, and \"other\" for all the rest\n", "\n", @@ -96,22 +292,74 @@ "print(categorical['GENDER'].value_counts())\n", "categorical['GENDER'] = categorical['GENDER'].fillna('F')\n", "```" - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": null, - "id": "b583feaa", - "metadata": {}, + "source": [ + "data['GENDER'].value_counts()" + ], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "F 51277\n", + "M 39094\n", + " 2957\n", + "U 1715\n", + "J 365\n", + "C 2\n", + "A 2\n", + "Name: GENDER, dtype: int64" + ] + }, + "metadata": {}, + "execution_count": 31 + } + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "data['GENDER'] = data['GENDER'].fillna('F')" + ], "outputs": [], - "source": [] + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "data['GENDER'] = data['GENDER'].apply(lambda x: x if x in ['M', 'F'] else 'other')\n", + "data['GENDER'].value_counts()" + ], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "F 51277\n", + "M 39094\n", + "other 5041\n", + "Name: GENDER, dtype: int64" + ] + }, + "metadata": {}, + "execution_count": 33 + } + ], + "metadata": {} } ], "metadata": { "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" + "name": "python3", + "display_name": "Python 3.9.7 64-bit" }, "language_info": { "codemirror_mode": { @@ -123,9 +371,12 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.9.7" + }, + "interpreter": { + "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49" } }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file