diff --git a/Solutions.ipynb b/Solutions.ipynb index 89aa338..ebda5ae 100644 --- a/Solutions.ipynb +++ b/Solutions.ipynb @@ -16,6 +16,396 @@ "In this lab, you will use learningSet.csv file which you already have cloned in today's activities." ] }, + { + "cell_type": "code", + "execution_count": 2, + "id": "515a6a84", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import datetime\n", + "import warnings\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "c65b3b74", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ODATEDWOSOURCETCODESTATEZIPMAILCODEPVASTATEDOBNOEXCHRECINHSE...TARGET_DHPHONE_DRFA_2RRFA_2FRFA_2AMDMAUD_RMDMAUD_FMDMAUD_ACLUSTER2GEOCODE2
08901GRI0IL6108137120...0.00L4EXXX39.0C
19401BOA1CA9132652020...0.00L2GXXX1.0A
29001AMH1NC2701700...0.01L4EXXX60.0C
38701BRY0CA9595328010...0.01L4EXXX41.0C
486010FL3317620010X...0.01L2FXXX26.0A
..................................................................
954079601ASE1AK9950400...0.00L1GXXX12.0C
954089601DCD1TX7737950010...0.01L1FXXX2.0A
954099501MBC1MI4891038010...0.01L3EXXX34.0B
954108601PRV0CA9132040050X...18.01L4FXXX11.0A
954118801MCC2NC2840918010X...0.01L1GC1C12.0C
\n", + "

95412 rows × 481 columns

\n", + "
" + ], + "text/plain": [ + " ODATEDW OSOURCE TCODE STATE ZIP MAILCODE PVASTATE DOB NOEXCH \\\n", + "0 8901 GRI 0 IL 61081 3712 0 \n", + "1 9401 BOA 1 CA 91326 5202 0 \n", + "2 9001 AMH 1 NC 27017 0 0 \n", + "3 8701 BRY 0 CA 95953 2801 0 \n", + "4 8601 0 FL 33176 2001 0 \n", + "... ... ... ... ... ... ... ... ... ... \n", + "95407 9601 ASE 1 AK 99504 0 0 \n", + "95408 9601 DCD 1 TX 77379 5001 0 \n", + "95409 9501 MBC 1 MI 48910 3801 0 \n", + "95410 8601 PRV 0 CA 91320 4005 0 \n", + "95411 8801 MCC 2 NC 28409 1801 0 \n", + "\n", + " RECINHSE ... TARGET_D HPHONE_D RFA_2R RFA_2F RFA_2A MDMAUD_R MDMAUD_F \\\n", + "0 ... 0.0 0 L 4 E X X \n", + "1 ... 0.0 0 L 2 G X X \n", + "2 ... 0.0 1 L 4 E X X \n", + "3 ... 0.0 1 L 4 E X X \n", + "4 X ... 0.0 1 L 2 F X X \n", + "... ... ... ... ... ... ... ... ... ... \n", + "95407 ... 0.0 0 L 1 G X X \n", + "95408 ... 0.0 1 L 1 F X X \n", + "95409 ... 0.0 1 L 3 E X X \n", + "95410 X ... 18.0 1 L 4 F X X \n", + "95411 X ... 0.0 1 L 1 G C 1 \n", + "\n", + " MDMAUD_A CLUSTER2 GEOCODE2 \n", + "0 X 39.0 C \n", + "1 X 1.0 A \n", + "2 X 60.0 C \n", + "3 X 41.0 C \n", + "4 X 26.0 A \n", + "... ... ... ... \n", + "95407 X 12.0 C \n", + "95408 X 2.0 A \n", + "95409 X 34.0 B \n", + "95410 X 11.0 A \n", + "95411 C 12.0 C \n", + "\n", + "[95412 rows x 481 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = pd.read_csv('learningSet.csv')\n", + "data" + ] + }, { "cell_type": "markdown", "id": "5bbb5252", @@ -26,11 +416,130 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "dbe69e8b", "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
percentage
RDATE_599.990567
RAMNT_599.990567
RDATE_399.746363
RAMNT_399.746363
RDATE_499.705488
......
ADATE_1621.343227
ADATE_1419.774242
ADATE_911.785729
ADATE_1110.923154
NEXTDATE10.452564
\n", + "

80 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " percentage\n", + "RDATE_5 99.990567\n", + "RAMNT_5 99.990567\n", + "RDATE_3 99.746363\n", + "RAMNT_3 99.746363\n", + "RDATE_4 99.705488\n", + "... ...\n", + "ADATE_16 21.343227\n", + "ADATE_14 19.774242\n", + "ADATE_9 11.785729\n", + "ADATE_11 10.923154\n", + "NEXTDATE 10.452564\n", + "\n", + "[80 rows x 1 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nulls = pd.DataFrame(data.isna().sum()*100/len(data), columns=['percentage'])\n", + "nulls.sort_values('percentage', ascending = False).head(80)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "bfc5a339", + "metadata": {}, "outputs": [], - "source": [] + "source": [ + "drop_col= list(nulls[nulls['percentage'] > 10].index)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "9139b8ab", + "metadata": {}, + "outputs": [], + "source": [ + "new_data = data.drop(columns=drop_col, axis=1)" + ] }, { "cell_type": "markdown", @@ -45,11 +554,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "3a07b75a", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "drop_list = new_data.drop(['ZIP', 'OSOURCE'], axis=1)" + ] }, { "cell_type": "markdown", @@ -61,11 +572,134 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "2b6f01b6", + "execution_count": 8, + "id": "21f6269e", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "null_percent_data = data.isna().sum().sort_values(ascending = False)*100/len(data)\n", + "null_percent_data = null_percent_data.reset_index()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "2b6f01b6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
column_namenull_percent
0RDATE_599.990567
1RAMNT_599.990567
2RDATE_399.746363
3RAMNT_399.746363
4RDATE_499.705488
.........
72AGE24.802960
73INCOME22.309563
74ADATE_1822.285457
75ADATE_1621.343227
76ADATE_1419.774242
\n", + "

77 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " column_name null_percent\n", + "0 RDATE_5 99.990567\n", + "1 RAMNT_5 99.990567\n", + "2 RDATE_3 99.746363\n", + "3 RAMNT_3 99.746363\n", + "4 RDATE_4 99.705488\n", + ".. ... ...\n", + "72 AGE 24.802960\n", + "73 INCOME 22.309563\n", + "74 ADATE_18 22.285457\n", + "75 ADATE_16 21.343227\n", + "76 ADATE_14 19.774242\n", + "\n", + "[77 rows x 2 columns]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "null_percent_data.columns = ['column_name', 'null_percent']\n", + "columns_above_85 = null_percent_data[null_percent_data['null_percent'] > 15]\n", + "columns_above_85" + ] }, { "cell_type": "markdown", @@ -77,11 +711,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "ca6c4d91", "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "ename": "TypeError", + "evalue": "cannot concatenate object of type ''; only Series and DataFrame objs are valid", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcolumns_above_85\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'column_name'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mdrop_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mappend\u001b[0;34m(self, other, ignore_index, verify_integrity, sort)\u001b[0m\n\u001b[1;32m 7980\u001b[0m \u001b[0mto_concat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7981\u001b[0m return (\n\u001b[0;32m-> 7982\u001b[0;31m concat(\n\u001b[0m\u001b[1;32m 7983\u001b[0m \u001b[0mto_concat\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7984\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mignore_index\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/pandas/core/reshape/concat.py\u001b[0m in \u001b[0;36mconcat\u001b[0;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001b[0m\n\u001b[1;32m 283\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIndexes\u001b[0m \u001b[0mhave\u001b[0m \u001b[0moverlapping\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'a'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 284\u001b[0m \"\"\"\n\u001b[0;32m--> 285\u001b[0;31m op = _Concatenator(\n\u001b[0m\u001b[1;32m 286\u001b[0m \u001b[0mobjs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 287\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/pandas/core/reshape/concat.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, objs, axis, join, keys, levels, names, ignore_index, verify_integrity, copy, sort)\u001b[0m\n\u001b[1;32m 368\u001b[0m \u001b[0;34m\"only Series and DataFrame objs are valid\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 369\u001b[0m )\n\u001b[0;32m--> 370\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 371\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 372\u001b[0m \u001b[0;31m# consolidate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: cannot concatenate object of type ''; only Series and DataFrame objs are valid" + ] + } + ], + "source": [ + "for i in columns_above_85['column_name']:\n", + " drop_list.append(i)" + ] }, { "cell_type": "markdown", @@ -123,7 +775,20 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.8.8" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false } }, "nbformat": 4,