diff --git a/lab-revisiting-machine-learning [ignacio].ipynb b/lab-revisiting-machine-learning [ignacio].ipynb new file mode 100644 index 0000000..e9a1853 --- /dev/null +++ b/lab-revisiting-machine-learning [ignacio].ipynb @@ -0,0 +1,567 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "ca778f3d", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "id": "bf32ae3b", + "metadata": {}, + "source": [ + "# Lab | Revisiting Machine Learning Case Study" + ] + }, + { + "cell_type": "markdown", + "id": "73df37d0", + "metadata": {}, + "source": [ + "In this lab, you will use learningSet.csv file which you already have cloned in today's activities." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "dbe69e8b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/IPython/core/interactiveshell.py:3169: DtypeWarning: Columns (8) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n" + ] + } + ], + "source": [ + "data = pd.read_csv(\"learningSet.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "77a8676f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(95412, 481)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.shape" + ] + }, + { + "cell_type": "markdown", + "id": "5bbb5252", + "metadata": {}, + "source": [ + "### 1. Check for null values in all the columns" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "22285efa", + "metadata": {}, + "outputs": [], + "source": [ + "def checking_nulls(df):\n", + " # This function shows which columns have null values and returns a df with only nulls\n", + " nulls_lst = []\n", + " cols_lst = []\n", + " for c in df.columns:\n", + " nulls = (int(df[c].isnull().sum()) / int(len(df[c])))*100\n", + " if nulls > 0:\n", + " nulls_lst.append(nulls)\n", + " cols_lst.append(c) \n", + " nulls_dict = dict(zip(cols_lst,nulls_lst))\n", + " nulls_df = pd.DataFrame([nulls_dict]).transpose()\n", + " nulls_df.columns = ['n_nulls']\n", + " nulls_df = nulls_df.sort_values(by = ['n_nulls'], ascending = [False])\n", + " return nulls_df" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "fe0f6bf7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
n_nulls
RDATE_599.990567
RAMNT_599.990567
RAMNT_399.746363
RDATE_399.746363
RDATE_499.705488
......
MSA0.138347
ADI0.138347
DMA0.138347
CLUSTER20.138347
GEOCODE20.138347
\n", + "

92 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " n_nulls\n", + "RDATE_5 99.990567\n", + "RAMNT_5 99.990567\n", + "RAMNT_3 99.746363\n", + "RDATE_3 99.746363\n", + "RDATE_4 99.705488\n", + "... ...\n", + "MSA 0.138347\n", + "ADI 0.138347\n", + "DMA 0.138347\n", + "CLUSTER2 0.138347\n", + "GEOCODE2 0.138347\n", + "\n", + "[92 rows x 1 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "checking_nulls(data)" + ] + }, + { + "cell_type": "markdown", + "id": "6febccdc", + "metadata": {}, + "source": [ + "### 2. Exclude the following variables by looking at the definitions. Create a new empty list called drop_list. We will append this list and then drop all the columns in this list later:\n", + "\n", + "- `OSOURCE` - symbol definitions not provided, too many categories\n", + "- `ZIP CODE` - we are including state already\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3a07b75a", + "metadata": {}, + "outputs": [], + "source": [ + "drop_list = ['OSOURCE', 'ZIP CODE']" + ] + }, + { + "cell_type": "markdown", + "id": "6a2470e4", + "metadata": {}, + "source": [ + "### 3. Identify columns that over 85% missing values" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "2b6f01b6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
n_nulls
RDATE_599.990567
RAMNT_599.990567
RAMNT_399.746363
RDATE_399.746363
RDATE_499.705488
RAMNT_499.705488
RDATE_699.186685
RAMNT_699.186685
RDATE_1592.388798
RAMNT_1592.388798
RAMNT_2391.763091
RDATE_2391.763091
RDATE_2091.732696
RAMNT_2091.732696
RAMNT_790.677273
RDATE_790.677273
RDATE_1790.146942
RAMNT_1790.146942
RDATE_2190.029556
RAMNT_2190.029556
RDATE_1089.035970
RAMNT_1089.035970
RDATE_1387.160944
RAMNT_1387.160944
NUMCHLD87.018404
\n", + "
" + ], + "text/plain": [ + " n_nulls\n", + "RDATE_5 99.990567\n", + "RAMNT_5 99.990567\n", + "RAMNT_3 99.746363\n", + "RDATE_3 99.746363\n", + "RDATE_4 99.705488\n", + "RAMNT_4 99.705488\n", + "RDATE_6 99.186685\n", + "RAMNT_6 99.186685\n", + "RDATE_15 92.388798\n", + "RAMNT_15 92.388798\n", + "RAMNT_23 91.763091\n", + "RDATE_23 91.763091\n", + "RDATE_20 91.732696\n", + "RAMNT_20 91.732696\n", + "RAMNT_7 90.677273\n", + "RDATE_7 90.677273\n", + "RDATE_17 90.146942\n", + "RAMNT_17 90.146942\n", + "RDATE_21 90.029556\n", + "RAMNT_21 90.029556\n", + "RDATE_10 89.035970\n", + "RAMNT_10 89.035970\n", + "RDATE_13 87.160944\n", + "RAMNT_13 87.160944\n", + "NUMCHLD 87.018404" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nulls = checking_nulls(data)\n", + "nulls_85 = nulls[nulls['n_nulls']>85]\n", + "nulls_85" + ] + }, + { + "cell_type": "markdown", + "id": "70865946", + "metadata": {}, + "source": [ + "### 4. Remove those columns from the dataframe" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "1778034d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(95412, 481)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "ca6c4d91", + "metadata": {}, + "outputs": [], + "source": [ + "to_drop = []\n", + "for c in nulls_85.index:\n", + " to_drop.append(c)\n", + "data = data.drop(to_drop, axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "53647d9a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(95412, 456)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.shape" + ] + }, + { + "cell_type": "markdown", + "id": "7bbcf161", + "metadata": {}, + "source": [ + "### 5. Reduce the number of categories in the column GENDER. The column should only have either \"M\" for males, \"F\" for females, and \"other\" for all the rest\n", + "\n", + "Note that there are a few null values in the column. We will first replace those null values using the code below:\n", + "\n", + "```python\n", + "print(categorical['GENDER'].value_counts())\n", + "categorical['GENDER'] = categorical['GENDER'].fillna('F')\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "1ead0965", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "F 51277\n", + "M 39094\n", + " 2957\n", + "U 1715\n", + "J 365\n", + "C 2\n", + "A 2\n", + "Name: GENDER, dtype: int64\n" + ] + } + ], + "source": [ + "print(data['GENDER'].value_counts())\n", + "data['GENDER'] = data['GENDER'].fillna('F')" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "b583feaa", + "metadata": {}, + "outputs": [], + "source": [ + "data['GENDER'] = data['GENDER'].replace([' ', 'C', 'U', 'J', 'A'], \"other\")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "cc185d4c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['F', 'M', 'other'], dtype=object)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data['GENDER'].unique()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}