diff --git a/Solution.ipynb b/Solution.ipynb new file mode 100644 index 0000000..ca94e34 --- /dev/null +++ b/Solution.ipynb @@ -0,0 +1,2122 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import warnings\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "\n", + "\n", + "warnings.filterwarnings('ignore')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\Karo\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3146: DtypeWarning: Columns (8) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n" + ] + } + ], + "source": [ + "data = pd.read_csv('learningSet.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ODATEDWOSOURCETCODESTATEZIPMAILCODEPVASTATEDOBNOEXCHRECINHSE...TARGET_DHPHONE_DRFA_2RRFA_2FRFA_2AMDMAUD_RMDMAUD_FMDMAUD_ACLUSTER2GEOCODE2
08901GRI0IL6108137120...0.00L4EXXX39.0C
19401BOA1CA9132652020...0.00L2GXXX1.0A
29001AMH1NC2701700...0.01L4EXXX60.0C
38701BRY0CA9595328010...0.01L4EXXX41.0C
486010FL3317620010X...0.01L2FXXX26.0A
\n", + "

5 rows × 481 columns

\n", + "
" + ], + "text/plain": [ + " ODATEDW OSOURCE TCODE STATE ZIP MAILCODE PVASTATE DOB NOEXCH \\\n", + "0 8901 GRI 0 IL 61081 3712 0 \n", + "1 9401 BOA 1 CA 91326 5202 0 \n", + "2 9001 AMH 1 NC 27017 0 0 \n", + "3 8701 BRY 0 CA 95953 2801 0 \n", + "4 8601 0 FL 33176 2001 0 \n", + "\n", + " RECINHSE ... TARGET_D HPHONE_D RFA_2R RFA_2F RFA_2A MDMAUD_R MDMAUD_F \\\n", + "0 ... 0.0 0 L 4 E X X \n", + "1 ... 0.0 0 L 2 G X X \n", + "2 ... 0.0 1 L 4 E X X \n", + "3 ... 0.0 1 L 4 E X X \n", + "4 X ... 0.0 1 L 2 F X X \n", + "\n", + " MDMAUD_A CLUSTER2 GEOCODE2 \n", + "0 X 39.0 C \n", + "1 X 1.0 A \n", + "2 X 60.0 C \n", + "3 X 41.0 C \n", + "4 X 26.0 A \n", + "\n", + "[5 rows x 481 columns]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['ODATEDW',\n", + " 'OSOURCE',\n", + " 'TCODE',\n", + " 'STATE',\n", + " 'ZIP',\n", + " 'MAILCODE',\n", + " 'PVASTATE',\n", + " 'DOB',\n", + " 'NOEXCH',\n", + " 'RECINHSE',\n", + " 'RECP3',\n", + " 'RECPGVG',\n", + " 'RECSWEEP',\n", + " 'MDMAUD',\n", + " 'DOMAIN',\n", + " 'CLUSTER',\n", + " 'AGE',\n", + " 'AGEFLAG',\n", + " 'HOMEOWNR',\n", + " 'CHILD03',\n", + " 'CHILD07',\n", + " 'CHILD12',\n", + " 'CHILD18',\n", + " 'NUMCHLD',\n", + " 'INCOME',\n", + " 'GENDER',\n", + " 'WEALTH1',\n", + " 'HIT',\n", + " 'MBCRAFT',\n", + " 'MBGARDEN',\n", + " 'MBBOOKS',\n", + " 'MBCOLECT',\n", + " 'MAGFAML',\n", + " 'MAGFEM',\n", + " 'MAGMALE',\n", + " 'PUBGARDN',\n", + " 'PUBCULIN',\n", + " 'PUBHLTH',\n", + " 'PUBDOITY',\n", + " 'PUBNEWFN',\n", + " 'PUBPHOTO',\n", + " 'PUBOPP',\n", + " 'DATASRCE',\n", + " 'MALEMILI',\n", + " 'MALEVET',\n", + " 'VIETVETS',\n", + " 'WWIIVETS',\n", + " 'LOCALGOV',\n", + " 'STATEGOV',\n", + " 'FEDGOV',\n", + " 'SOLP3',\n", + " 'SOLIH',\n", + " 'MAJOR',\n", + " 'WEALTH2',\n", + " 'GEOCODE',\n", + " 'COLLECT1',\n", + " 'VETERANS',\n", + " 'BIBLE',\n", + " 'CATLG',\n", + " 'HOMEE',\n", + " 'PETS',\n", + " 'CDPLAY',\n", + " 'STEREO',\n", + " 'PCOWNERS',\n", + " 'PHOTO',\n", + " 'CRAFTS',\n", + " 'FISHER',\n", + " 'GARDENIN',\n", + " 'BOATS',\n", + " 'WALKER',\n", + " 'KIDSTUFF',\n", + " 'CARDS',\n", + " 'PLATES',\n", + " 'LIFESRC',\n", + " 'PEPSTRFL',\n", + " 'POP901',\n", + " 'POP902',\n", + " 'POP903',\n", + " 'POP90C1',\n", + " 'POP90C2',\n", + " 'POP90C3',\n", + " 'POP90C4',\n", + " 'POP90C5',\n", + " 'ETH1',\n", + " 'ETH2',\n", + " 'ETH3',\n", + " 'ETH4',\n", + " 'ETH5',\n", + " 'ETH6',\n", + " 'ETH7',\n", + " 'ETH8',\n", + " 'ETH9',\n", + " 'ETH10',\n", + " 'ETH11',\n", + " 'ETH12',\n", + " 'ETH13',\n", + " 'ETH14',\n", + " 'ETH15',\n", + " 'ETH16',\n", + " 'AGE901',\n", + " 'AGE902',\n", + " 'AGE903',\n", + " 'AGE904',\n", + " 'AGE905',\n", + " 'AGE906',\n", + " 'AGE907',\n", + " 'CHIL1',\n", + " 'CHIL2',\n", + " 'CHIL3',\n", + " 'AGEC1',\n", + " 'AGEC2',\n", + " 'AGEC3',\n", + " 'AGEC4',\n", + " 'AGEC5',\n", + " 'AGEC6',\n", + " 'AGEC7',\n", + " 'CHILC1',\n", + " 'CHILC2',\n", + " 'CHILC3',\n", + " 'CHILC4',\n", + " 'CHILC5',\n", + " 'HHAGE1',\n", + " 'HHAGE2',\n", + " 'HHAGE3',\n", + " 'HHN1',\n", + " 'HHN2',\n", + " 'HHN3',\n", + " 'HHN4',\n", + " 'HHN5',\n", + " 'HHN6',\n", + " 'MARR1',\n", + " 'MARR2',\n", + " 'MARR3',\n", + " 'MARR4',\n", + " 'HHP1',\n", + " 'HHP2',\n", + " 'DW1',\n", + " 'DW2',\n", + " 'DW3',\n", + " 'DW4',\n", + " 'DW5',\n", + " 'DW6',\n", + " 'DW7',\n", + " 'DW8',\n", + " 'DW9',\n", + " 'HV1',\n", + " 'HV2',\n", + " 'HV3',\n", + " 'HV4',\n", + " 'HU1',\n", + " 'HU2',\n", + " 'HU3',\n", + " 'HU4',\n", + " 'HU5',\n", + " 'HHD1',\n", + " 'HHD2',\n", + " 'HHD3',\n", + " 'HHD4',\n", + " 'HHD5',\n", + " 'HHD6',\n", + " 'HHD7',\n", + " 'HHD8',\n", + " 'HHD9',\n", + " 'HHD10',\n", + " 'HHD11',\n", + " 'HHD12',\n", + " 'ETHC1',\n", + " 'ETHC2',\n", + " 'ETHC3',\n", + " 'ETHC4',\n", + " 'ETHC5',\n", + " 'ETHC6',\n", + " 'HVP1',\n", + " 'HVP2',\n", + " 'HVP3',\n", + " 'HVP4',\n", + " 'HVP5',\n", + " 'HVP6',\n", + " 'HUR1',\n", + " 'HUR2',\n", + " 'RHP1',\n", + " 'RHP2',\n", + " 'RHP3',\n", + " 'RHP4',\n", + " 'HUPA1',\n", + " 'HUPA2',\n", + " 'HUPA3',\n", + " 'HUPA4',\n", + " 'HUPA5',\n", + " 'HUPA6',\n", + " 'HUPA7',\n", + " 'RP1',\n", + " 'RP2',\n", + " 'RP3',\n", + " 'RP4',\n", + " 'MSA',\n", + " 'ADI',\n", + " 'DMA',\n", + " 'IC1',\n", + " 'IC2',\n", + " 'IC3',\n", + " 'IC4',\n", + " 'IC5',\n", + " 'IC6',\n", + " 'IC7',\n", + " 'IC8',\n", + " 'IC9',\n", + " 'IC10',\n", + " 'IC11',\n", + " 'IC12',\n", + " 'IC13',\n", + " 'IC14',\n", + " 'IC15',\n", + " 'IC16',\n", + " 'IC17',\n", + " 'IC18',\n", + " 'IC19',\n", + " 'IC20',\n", + " 'IC21',\n", + " 'IC22',\n", + " 'IC23',\n", + " 'HHAS1',\n", + " 'HHAS2',\n", + " 'HHAS3',\n", + " 'HHAS4',\n", + " 'MC1',\n", + " 'MC2',\n", + " 'MC3',\n", + " 'TPE1',\n", + " 'TPE2',\n", + " 'TPE3',\n", + " 'TPE4',\n", + " 'TPE5',\n", + " 'TPE6',\n", + " 'TPE7',\n", + " 'TPE8',\n", + " 'TPE9',\n", + " 'PEC1',\n", + " 'PEC2',\n", + " 'TPE10',\n", + " 'TPE11',\n", + " 'TPE12',\n", + " 'TPE13',\n", + " 'LFC1',\n", + " 'LFC2',\n", + " 'LFC3',\n", + " 'LFC4',\n", + " 'LFC5',\n", + " 'LFC6',\n", + " 'LFC7',\n", + " 'LFC8',\n", + " 'LFC9',\n", + " 'LFC10',\n", + " 'OCC1',\n", + " 'OCC2',\n", + " 'OCC3',\n", + " 'OCC4',\n", + " 'OCC5',\n", + " 'OCC6',\n", + " 'OCC7',\n", + " 'OCC8',\n", + " 'OCC9',\n", + " 'OCC10',\n", + " 'OCC11',\n", + " 'OCC12',\n", + " 'OCC13',\n", + " 'EIC1',\n", + " 'EIC2',\n", + " 'EIC3',\n", + " 'EIC4',\n", + " 'EIC5',\n", + " 'EIC6',\n", + " 'EIC7',\n", + " 'EIC8',\n", + " 'EIC9',\n", + " 'EIC10',\n", + " 'EIC11',\n", + " 'EIC12',\n", + " 'EIC13',\n", + " 'EIC14',\n", + " 'EIC15',\n", + " 'EIC16',\n", + " 'OEDC1',\n", + " 'OEDC2',\n", + " 'OEDC3',\n", + " 'OEDC4',\n", + " 'OEDC5',\n", + " 'OEDC6',\n", + " 'OEDC7',\n", + " 'EC1',\n", + " 'EC2',\n", + " 'EC3',\n", + " 'EC4',\n", + " 'EC5',\n", + " 'EC6',\n", + " 'EC7',\n", + " 'EC8',\n", + " 'SEC1',\n", + " 'SEC2',\n", + " 'SEC3',\n", + " 'SEC4',\n", + " 'SEC5',\n", + " 'AFC1',\n", + " 'AFC2',\n", + " 'AFC3',\n", + " 'AFC4',\n", + " 'AFC5',\n", + " 'AFC6',\n", + " 'VC1',\n", + " 'VC2',\n", + " 'VC3',\n", + " 'VC4',\n", + " 'ANC1',\n", + " 'ANC2',\n", + " 'ANC3',\n", + " 'ANC4',\n", + " 'ANC5',\n", + " 'ANC6',\n", + " 'ANC7',\n", + " 'ANC8',\n", + " 'ANC9',\n", + " 'ANC10',\n", + " 'ANC11',\n", + " 'ANC12',\n", + " 'ANC13',\n", + " 'ANC14',\n", + " 'ANC15',\n", + " 'POBC1',\n", + " 'POBC2',\n", + " 'LSC1',\n", + " 'LSC2',\n", + " 'LSC3',\n", + " 'LSC4',\n", + " 'VOC1',\n", + " 'VOC2',\n", + " 'VOC3',\n", + " 'HC1',\n", + " 'HC2',\n", + " 'HC3',\n", + " 'HC4',\n", + " 'HC5',\n", + " 'HC6',\n", + " 'HC7',\n", + " 'HC8',\n", + " 'HC9',\n", + " 'HC10',\n", + " 'HC11',\n", + " 'HC12',\n", + " 'HC13',\n", + " 'HC14',\n", + " 'HC15',\n", + " 'HC16',\n", + " 'HC17',\n", + " 'HC18',\n", + " 'HC19',\n", + " 'HC20',\n", + " 'HC21',\n", + " 'MHUC1',\n", + " 'MHUC2',\n", + " 'AC1',\n", + " 'AC2',\n", + " 'ADATE_2',\n", + " 'ADATE_3',\n", + " 'ADATE_4',\n", + " 'ADATE_5',\n", + " 'ADATE_6',\n", + " 'ADATE_7',\n", + " 'ADATE_8',\n", + " 'ADATE_9',\n", + " 'ADATE_10',\n", + " 'ADATE_11',\n", + " 'ADATE_12',\n", + " 'ADATE_13',\n", + " 'ADATE_14',\n", + " 'ADATE_15',\n", + " 'ADATE_16',\n", + " 'ADATE_17',\n", + " 'ADATE_18',\n", + " 'ADATE_19',\n", + " 'ADATE_20',\n", + " 'ADATE_21',\n", + " 'ADATE_22',\n", + " 'ADATE_23',\n", + " 'ADATE_24',\n", + " 'RFA_2',\n", + " 'RFA_3',\n", + " 'RFA_4',\n", + " 'RFA_5',\n", + " 'RFA_6',\n", + " 'RFA_7',\n", + " 'RFA_8',\n", + " 'RFA_9',\n", + " 'RFA_10',\n", + " 'RFA_11',\n", + " 'RFA_12',\n", + " 'RFA_13',\n", + " 'RFA_14',\n", + " 'RFA_15',\n", + " 'RFA_16',\n", + " 'RFA_17',\n", + " 'RFA_18',\n", + " 'RFA_19',\n", + " 'RFA_20',\n", + " 'RFA_21',\n", + " 'RFA_22',\n", + " 'RFA_23',\n", + " 'RFA_24',\n", + " 'CARDPROM',\n", + " 'MAXADATE',\n", + " 'NUMPROM',\n", + " 'CARDPM12',\n", + " 'NUMPRM12',\n", + " 'RDATE_3',\n", + " 'RDATE_4',\n", + " 'RDATE_5',\n", + " 'RDATE_6',\n", + " 'RDATE_7',\n", + " 'RDATE_8',\n", + " 'RDATE_9',\n", + " 'RDATE_10',\n", + " 'RDATE_11',\n", + " 'RDATE_12',\n", + " 'RDATE_13',\n", + " 'RDATE_14',\n", + " 'RDATE_15',\n", + " 'RDATE_16',\n", + " 'RDATE_17',\n", + " 'RDATE_18',\n", + " 'RDATE_19',\n", + " 'RDATE_20',\n", + " 'RDATE_21',\n", + " 'RDATE_22',\n", + " 'RDATE_23',\n", + " 'RDATE_24',\n", + " 'RAMNT_3',\n", + " 'RAMNT_4',\n", + " 'RAMNT_5',\n", + " 'RAMNT_6',\n", + " 'RAMNT_7',\n", + " 'RAMNT_8',\n", + " 'RAMNT_9',\n", + " 'RAMNT_10',\n", + " 'RAMNT_11',\n", + " 'RAMNT_12',\n", + " 'RAMNT_13',\n", + " 'RAMNT_14',\n", + " 'RAMNT_15',\n", + " 'RAMNT_16',\n", + " 'RAMNT_17',\n", + " 'RAMNT_18',\n", + " 'RAMNT_19',\n", + " 'RAMNT_20',\n", + " 'RAMNT_21',\n", + " 'RAMNT_22',\n", + " 'RAMNT_23',\n", + " 'RAMNT_24',\n", + " 'RAMNTALL',\n", + " 'NGIFTALL',\n", + " 'CARDGIFT',\n", + " 'MINRAMNT',\n", + " 'MINRDATE',\n", + " 'MAXRAMNT',\n", + " 'MAXRDATE',\n", + " 'LASTGIFT',\n", + " 'LASTDATE',\n", + " 'FISTDATE',\n", + " 'NEXTDATE',\n", + " 'TIMELAG',\n", + " 'AVGGIFT',\n", + " 'CONTROLN',\n", + " 'TARGET_B',\n", + " 'TARGET_D',\n", + " 'HPHONE_D',\n", + " 'RFA_2R',\n", + " 'RFA_2F',\n", + " 'RFA_2A',\n", + " 'MDMAUD_R',\n", + " 'MDMAUD_F',\n", + " 'MDMAUD_A',\n", + " 'CLUSTER2',\n", + " 'GEOCODE2']" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(data.columns)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TARGET_BTARGET_D
000.0
100.0
200.0
300.0
400.0
.........
9540700.0
9540800.0
9540900.0
95410118.0
9541100.0
\n", + "

95412 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " TARGET_B TARGET_D\n", + "0 0 0.0\n", + "1 0 0.0\n", + "2 0 0.0\n", + "3 0 0.0\n", + "4 0 0.0\n", + "... ... ...\n", + "95407 0 0.0\n", + "95408 0 0.0\n", + "95409 0 0.0\n", + "95410 1 18.0\n", + "95411 0 0.0\n", + "\n", + "[95412 rows x 2 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data[['TARGET_B','TARGET_D']]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
percentage
RAMNT_599.990567
RDATE_599.990567
RAMNT_399.746363
RDATE_399.746363
RAMNT_499.705488
......
EIC130.000000
EIC140.000000
EIC150.000000
EIC160.000000
ODATEDW0.000000
\n", + "

481 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " percentage\n", + "RAMNT_5 99.990567\n", + "RDATE_5 99.990567\n", + "RAMNT_3 99.746363\n", + "RDATE_3 99.746363\n", + "RAMNT_4 99.705488\n", + "... ...\n", + "EIC13 0.000000\n", + "EIC14 0.000000\n", + "EIC15 0.000000\n", + "EIC16 0.000000\n", + "ODATEDW 0.000000\n", + "\n", + "[481 rows x 1 columns]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#nulls= (data.isna().sum().sort_values(ascending = False)/len(data)*100)\n", + "#nulls\n", + "nulls = pd.DataFrame(data.isna().sum().sort_values(ascending = False)/len(data)*100, columns=['percentage'])\n", + "nulls" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "drop_col= list(nulls[nulls['percentage'] > 10].index)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "81" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(drop_col)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ODATEDWOSOURCETCODESTATEZIPMAILCODEPVASTATEDOBNOEXCHRECINHSE...TARGET_DHPHONE_DRFA_2RRFA_2FRFA_2AMDMAUD_RMDMAUD_FMDMAUD_ACLUSTER2GEOCODE2
08901GRI0IL6108137120...0.00L4EXXX39.0C
19401BOA1CA9132652020...0.00L2GXXX1.0A
29001AMH1NC2701700...0.01L4EXXX60.0C
38701BRY0CA9595328010...0.01L4EXXX41.0C
486010FL3317620010X...0.01L2FXXX26.0A
..................................................................
954079601ASE1AK9950400...0.00L1GXXX12.0C
954089601DCD1TX7737950010...0.01L1FXXX2.0A
954099501MBC1MI4891038010...0.01L3EXXX34.0B
954108601PRV0CA9132040050X...18.01L4FXXX11.0A
954118801MCC2NC2840918010X...0.01L1GC1C12.0C
\n", + "

95412 rows × 400 columns

\n", + "
" + ], + "text/plain": [ + " ODATEDW OSOURCE TCODE STATE ZIP MAILCODE PVASTATE DOB NOEXCH \\\n", + "0 8901 GRI 0 IL 61081 3712 0 \n", + "1 9401 BOA 1 CA 91326 5202 0 \n", + "2 9001 AMH 1 NC 27017 0 0 \n", + "3 8701 BRY 0 CA 95953 2801 0 \n", + "4 8601 0 FL 33176 2001 0 \n", + "... ... ... ... ... ... ... ... ... ... \n", + "95407 9601 ASE 1 AK 99504 0 0 \n", + "95408 9601 DCD 1 TX 77379 5001 0 \n", + "95409 9501 MBC 1 MI 48910 3801 0 \n", + "95410 8601 PRV 0 CA 91320 4005 0 \n", + "95411 8801 MCC 2 NC 28409 1801 0 \n", + "\n", + " RECINHSE ... TARGET_D HPHONE_D RFA_2R RFA_2F RFA_2A MDMAUD_R MDMAUD_F \\\n", + "0 ... 0.0 0 L 4 E X X \n", + "1 ... 0.0 0 L 2 G X X \n", + "2 ... 0.0 1 L 4 E X X \n", + "3 ... 0.0 1 L 4 E X X \n", + "4 X ... 0.0 1 L 2 F X X \n", + "... ... ... ... ... ... ... ... ... ... \n", + "95407 ... 0.0 0 L 1 G X X \n", + "95408 ... 0.0 1 L 1 F X X \n", + "95409 ... 0.0 1 L 3 E X X \n", + "95410 X ... 18.0 1 L 4 F X X \n", + "95411 X ... 0.0 1 L 1 G C 1 \n", + "\n", + " MDMAUD_A CLUSTER2 GEOCODE2 \n", + "0 X 39.0 C \n", + "1 X 1.0 A \n", + "2 X 60.0 C \n", + "3 X 41.0 C \n", + "4 X 26.0 A \n", + "... ... ... ... \n", + "95407 X 12.0 C \n", + "95408 X 2.0 A \n", + "95409 X 34.0 B \n", + "95410 X 11.0 A \n", + "95411 C 12.0 C \n", + "\n", + "[95412 rows x 400 columns]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data1 = data.drop(columns=drop_col, axis=1)\n", + "data1.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ODATEDWTCODESTATEMAILCODEPVASTATEDOBNOEXCHRECINHSERECP3RECPGVG...TARGET_DHPHONE_DRFA_2RRFA_2FRFA_2AMDMAUD_RMDMAUD_FMDMAUD_ACLUSTER2GEOCODE2
089010IL37120...0.00L4EXXX39.0C
194011CA52020...0.00L2GXXX1.0A
290011NC00...0.01L4EXXX60.0C
387010CA28010...0.01L4EXXX41.0C
486010FL20010XX...0.01L2FXXX26.0A
\n", + "

5 rows × 398 columns

\n", + "
" + ], + "text/plain": [ + " ODATEDW TCODE STATE MAILCODE PVASTATE DOB NOEXCH RECINHSE RECP3 RECPGVG \\\n", + "0 8901 0 IL 3712 0 \n", + "1 9401 1 CA 5202 0 \n", + "2 9001 1 NC 0 0 \n", + "3 8701 0 CA 2801 0 \n", + "4 8601 0 FL 2001 0 X X \n", + "\n", + " ... TARGET_D HPHONE_D RFA_2R RFA_2F RFA_2A MDMAUD_R MDMAUD_F MDMAUD_A \\\n", + "0 ... 0.0 0 L 4 E X X X \n", + "1 ... 0.0 0 L 2 G X X X \n", + "2 ... 0.0 1 L 4 E X X X \n", + "3 ... 0.0 1 L 4 E X X X \n", + "4 ... 0.0 1 L 2 F X X X \n", + "\n", + " CLUSTER2 GEOCODE2 \n", + "0 39.0 C \n", + "1 1.0 A \n", + "2 60.0 C \n", + "3 41.0 C \n", + "4 26.0 A \n", + "\n", + "[5 rows x 398 columns]" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data1 = data1.drop(['ZIP', 'OSOURCE'], axis=1)\n", + "data1.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "ADATE_12 9.352073\n", + "ADATE_7 9.300717\n", + "ADATE_6 3.728043\n", + "ADATE_8 3.679831\n", + "ADATE_4 2.296357\n", + " ... \n", + "OEDC3 0.000000\n", + "OEDC4 0.000000\n", + "OEDC5 0.000000\n", + "OEDC6 0.000000\n", + "ODATEDW 0.000000\n", + "Length: 398, dtype: float64" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(data1.isna().sum().sort_values(ascending = False)/len(data)*100)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ODATEDWTCODEDOBHITMALEMILIMALEVETVIETVETSWWIIVETSLOCALGOVSTATEGOV...LASTGIFTLASTDATEFISTDATEAVGGIFTCONTROLNTARGET_BTARGET_DHPHONE_DRFA_2FCLUSTER2
089010371200393418102...10.0951289117.7419359551500.00439.0
194011520216015551162...25.09512931015.66666714853500.0021.0
29001102020293368...5.0951290017.4814811507800.01460.0
38701028012023143130...10.0951287026.81250017255600.01441.0
486010200160128953263...15.0960179036.864865711200.01226.0
\n", + "

5 rows × 326 columns

\n", + "
" + ], + "text/plain": [ + " ODATEDW TCODE DOB HIT MALEMILI MALEVET VIETVETS WWIIVETS LOCALGOV \\\n", + "0 8901 0 3712 0 0 39 34 18 10 \n", + "1 9401 1 5202 16 0 15 55 11 6 \n", + "2 9001 1 0 2 0 20 29 33 6 \n", + "3 8701 0 2801 2 0 23 14 31 3 \n", + "4 8601 0 2001 60 1 28 9 53 26 \n", + "\n", + " STATEGOV ... LASTGIFT LASTDATE FISTDATE AVGGIFT CONTROLN TARGET_B \\\n", + "0 2 ... 10.0 9512 8911 7.741935 95515 0 \n", + "1 2 ... 25.0 9512 9310 15.666667 148535 0 \n", + "2 8 ... 5.0 9512 9001 7.481481 15078 0 \n", + "3 0 ... 10.0 9512 8702 6.812500 172556 0 \n", + "4 3 ... 15.0 9601 7903 6.864865 7112 0 \n", + "\n", + " TARGET_D HPHONE_D RFA_2F CLUSTER2 \n", + "0 0.0 0 4 39.0 \n", + "1 0.0 0 2 1.0 \n", + "2 0.0 1 4 60.0 \n", + "3 0.0 1 4 41.0 \n", + "4 0.0 1 2 26.0 \n", + "\n", + "[5 rows x 326 columns]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n", + "num = data1.select_dtypes(include=numerics)\n", + "num.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
STATEMAILCODEPVASTATENOEXCHRECINHSERECP3RECPGVGRECSWEEPMDMAUDDOMAIN...RFA_21RFA_22RFA_23RFA_24RFA_2RRFA_2AMDMAUD_RMDMAUD_FMDMAUD_AGEOCODE2
0IL0XXXXT2...S4ES4ES4ES4ELEXXXC
1CA0XXXXS1...N1EN1EF1ELGXXXA
2NC0XXXXR2...S4DS4DS3DLEXXXC
3CA0XXXXR2...A1DA1DLEXXXC
4FL0XXXXXXS2...A3DI4EA3DA3DLFXXXA
\n", + "

5 rows × 72 columns

\n", + "
" + ], + "text/plain": [ + " STATE MAILCODE PVASTATE NOEXCH RECINHSE RECP3 RECPGVG RECSWEEP MDMAUD \\\n", + "0 IL 0 XXXX \n", + "1 CA 0 XXXX \n", + "2 NC 0 XXXX \n", + "3 CA 0 XXXX \n", + "4 FL 0 X X XXXX \n", + "\n", + " DOMAIN ... RFA_21 RFA_22 RFA_23 RFA_24 RFA_2R RFA_2A MDMAUD_R MDMAUD_F \\\n", + "0 T2 ... S4E S4E S4E S4E L E X X \n", + "1 S1 ... N1E N1E F1E L G X X \n", + "2 R2 ... S4D S4D S3D L E X X \n", + "3 R2 ... A1D A1D L E X X \n", + "4 S2 ... A3D I4E A3D A3D L F X X \n", + "\n", + " MDMAUD_A GEOCODE2 \n", + "0 X C \n", + "1 X A \n", + "2 X C \n", + "3 X C \n", + "4 X A \n", + "\n", + "[5 rows x 72 columns]" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cat=data1.select_dtypes(exclude=numerics)\n", + "cat.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "F 51277\n", + "M 39094\n", + " 2957\n", + "U 1715\n", + "J 365\n", + "C 2\n", + "A 2\n", + "Name: GENDER, dtype: int64\n" + ] + } + ], + "source": [ + "print(cat['GENDER'].value_counts())\n", + "cat['GENDER'] = cat['GENDER'].fillna('F')" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "F 51277\n", + "M 39094\n", + "O 5041\n", + "Name: GENDER, dtype: int64" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def clean_gender(x):\n", + " if str(x).startswith('M'):\n", + " return 'M'\n", + " elif str(x).startswith('F'):\n", + " return 'F'\n", + " else:\n", + " return 'O'\n", + "\n", + "cat['GENDER'] = cat['GENDER'].apply(clean_gender)\n", + "\n", + "cat['GENDER'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/Solutions.ipynb b/Solutions.ipynb deleted file mode 100644 index 89aa338..0000000 --- a/Solutions.ipynb +++ /dev/null @@ -1,131 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "bf32ae3b", - "metadata": {}, - "source": [ - "# Lab | Revisiting Machine Learning Case Study" - ] - }, - { - "cell_type": "markdown", - "id": "73df37d0", - "metadata": {}, - "source": [ - "In this lab, you will use learningSet.csv file which you already have cloned in today's activities." - ] - }, - { - "cell_type": "markdown", - "id": "5bbb5252", - "metadata": {}, - "source": [ - "### 1. Check for null values in all the columns" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dbe69e8b", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "6febccdc", - "metadata": {}, - "source": [ - "### 2. Exclude the following variables by looking at the definitions. Create a new empty list called drop_list. We will append this list and then drop all the columns in this list later:\n", - "\n", - "- `OSOURCE` - symbol definitions not provided, too many categories\n", - "- `ZIP CODE` - we are including state already\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3a07b75a", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "6a2470e4", - "metadata": {}, - "source": [ - "### 3. Identify columns that over 85% missing values" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2b6f01b6", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "70865946", - "metadata": {}, - "source": [ - "### 4. Remove those columns from the dataframe" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ca6c4d91", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "7bbcf161", - "metadata": {}, - "source": [ - "### 5. Reduce the number of categories in the column GENDER. The column should only have either \"M\" for males, \"F\" for females, and \"other\" for all the rest\n", - "\n", - "Note that there are a few null values in the column. We will first replace those null values using the code below:\n", - "\n", - "```python\n", - "print(categorical['GENDER'].value_counts())\n", - "categorical['GENDER'] = categorical['GENDER'].fillna('F')\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b583feaa", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}