diff --git a/Solutions.ipynb b/Solutions.ipynb index 89aa338..3c9a48e 100644 --- a/Solutions.ipynb +++ b/Solutions.ipynb @@ -16,6 +16,1081 @@ "In this lab, you will use learningSet.csv file which you already have cloned in today's activities." ] }, + { + "cell_type": "code", + "execution_count": 1, + "id": "9fe58f79", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: xgboost in c:\\users\\claud\\anaconda3\\lib\\site-packages (1.4.2)\n", + "Requirement already satisfied: numpy in c:\\users\\claud\\anaconda3\\lib\\site-packages (from xgboost) (1.20.1)\n", + "Requirement already satisfied: scipy in c:\\users\\claud\\anaconda3\\lib\\site-packages (from xgboost) (1.6.2)\n" + ] + } + ], + "source": [ + "!pip install xgboost\n", + "\n", + "import pandas as pd\n", + "import numpy as np\n", + "import datetime\n", + "import warnings\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "import time\n", + "import pickle\n", + "\n", + "warnings.filterwarnings('ignore')\n", + "\n", + "from sklearn.model_selection import train_test_split, GridSearchCV\n", + "from sklearn.linear_model import LinearRegression, LogisticRegression\n", + "from sklearn.metrics import mean_absolute_error, mean_squared_error, classification_report\n", + "from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n", + "from sklearn.svm import SVR, SVC\n", + "from sklearn.preprocessing import StandardScaler, Normalizer\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.decomposition import PCA\n", + "from sklearn.feature_selection import VarianceThreshold, RFE\n", + "from xgboost import XGBRegressor, XGBClassifier\n", + "from imblearn.under_sampling import TomekLinks\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "b11c1aaf", + "metadata": {}, + "outputs": [], + "source": [ + "data = pd.read_csv(r'C:\\Users\\claud\\Downloads\\learningSet.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "c30ae880", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(95412, 481)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "0d4695ef", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['ODATEDW',\n", + " 'OSOURCE',\n", + " 'TCODE',\n", + " 'STATE',\n", + " 'ZIP',\n", + " 'MAILCODE',\n", + " 'PVASTATE',\n", + " 'DOB',\n", + " 'NOEXCH',\n", + " 'RECINHSE',\n", + " 'RECP3',\n", + " 'RECPGVG',\n", + " 'RECSWEEP',\n", + " 'MDMAUD',\n", + " 'DOMAIN',\n", + " 'CLUSTER',\n", + " 'AGE',\n", + " 'AGEFLAG',\n", + " 'HOMEOWNR',\n", + " 'CHILD03',\n", + " 'CHILD07',\n", + " 'CHILD12',\n", + " 'CHILD18',\n", + " 'NUMCHLD',\n", + " 'INCOME',\n", + " 'GENDER',\n", + " 'WEALTH1',\n", + " 'HIT',\n", + " 'MBCRAFT',\n", + " 'MBGARDEN',\n", + " 'MBBOOKS',\n", + " 'MBCOLECT',\n", + " 'MAGFAML',\n", + " 'MAGFEM',\n", + " 'MAGMALE',\n", + " 'PUBGARDN',\n", + " 'PUBCULIN',\n", + " 'PUBHLTH',\n", + " 'PUBDOITY',\n", + " 'PUBNEWFN',\n", + " 'PUBPHOTO',\n", + " 'PUBOPP',\n", + " 'DATASRCE',\n", + " 'MALEMILI',\n", + " 'MALEVET',\n", + " 'VIETVETS',\n", + " 'WWIIVETS',\n", + " 'LOCALGOV',\n", + " 'STATEGOV',\n", + " 'FEDGOV',\n", + " 'SOLP3',\n", + " 'SOLIH',\n", + " 'MAJOR',\n", + " 'WEALTH2',\n", + " 'GEOCODE',\n", + " 'COLLECT1',\n", + " 'VETERANS',\n", + " 'BIBLE',\n", + " 'CATLG',\n", + " 'HOMEE',\n", + " 'PETS',\n", + " 'CDPLAY',\n", + " 'STEREO',\n", + " 'PCOWNERS',\n", + " 'PHOTO',\n", + " 'CRAFTS',\n", + " 'FISHER',\n", + " 'GARDENIN',\n", + " 'BOATS',\n", + " 'WALKER',\n", + " 'KIDSTUFF',\n", + " 'CARDS',\n", + " 'PLATES',\n", + " 'LIFESRC',\n", + " 'PEPSTRFL',\n", + " 'POP901',\n", + " 'POP902',\n", + " 'POP903',\n", + " 'POP90C1',\n", + " 'POP90C2',\n", + " 'POP90C3',\n", + " 'POP90C4',\n", + " 'POP90C5',\n", + " 'ETH1',\n", + " 'ETH2',\n", + " 'ETH3',\n", + " 'ETH4',\n", + " 'ETH5',\n", + " 'ETH6',\n", + " 'ETH7',\n", + " 'ETH8',\n", + " 'ETH9',\n", + " 'ETH10',\n", + " 'ETH11',\n", + " 'ETH12',\n", + " 'ETH13',\n", + " 'ETH14',\n", + " 'ETH15',\n", + " 'ETH16',\n", + " 'AGE901',\n", + " 'AGE902',\n", + " 'AGE903',\n", + " 'AGE904',\n", + " 'AGE905',\n", + " 'AGE906',\n", + " 'AGE907',\n", + " 'CHIL1',\n", + " 'CHIL2',\n", + " 'CHIL3',\n", + " 'AGEC1',\n", + " 'AGEC2',\n", + " 'AGEC3',\n", + " 'AGEC4',\n", + " 'AGEC5',\n", + " 'AGEC6',\n", + " 'AGEC7',\n", + " 'CHILC1',\n", + " 'CHILC2',\n", + " 'CHILC3',\n", + " 'CHILC4',\n", + " 'CHILC5',\n", + " 'HHAGE1',\n", + " 'HHAGE2',\n", + " 'HHAGE3',\n", + " 'HHN1',\n", + " 'HHN2',\n", + " 'HHN3',\n", + " 'HHN4',\n", + " 'HHN5',\n", + " 'HHN6',\n", + " 'MARR1',\n", + " 'MARR2',\n", + " 'MARR3',\n", + " 'MARR4',\n", + " 'HHP1',\n", + " 'HHP2',\n", + " 'DW1',\n", + " 'DW2',\n", + " 'DW3',\n", + " 'DW4',\n", + " 'DW5',\n", + " 'DW6',\n", + " 'DW7',\n", + " 'DW8',\n", + " 'DW9',\n", + " 'HV1',\n", + " 'HV2',\n", + " 'HV3',\n", + " 'HV4',\n", + " 'HU1',\n", + " 'HU2',\n", + " 'HU3',\n", + " 'HU4',\n", + " 'HU5',\n", + " 'HHD1',\n", + " 'HHD2',\n", + " 'HHD3',\n", + " 'HHD4',\n", + " 'HHD5',\n", + " 'HHD6',\n", + " 'HHD7',\n", + " 'HHD8',\n", + " 'HHD9',\n", + " 'HHD10',\n", + " 'HHD11',\n", + " 'HHD12',\n", + " 'ETHC1',\n", + " 'ETHC2',\n", + " 'ETHC3',\n", + " 'ETHC4',\n", + " 'ETHC5',\n", + " 'ETHC6',\n", + " 'HVP1',\n", + " 'HVP2',\n", + " 'HVP3',\n", + " 'HVP4',\n", + " 'HVP5',\n", + " 'HVP6',\n", + " 'HUR1',\n", + " 'HUR2',\n", + " 'RHP1',\n", + " 'RHP2',\n", + " 'RHP3',\n", + " 'RHP4',\n", + " 'HUPA1',\n", + " 'HUPA2',\n", + " 'HUPA3',\n", + " 'HUPA4',\n", + " 'HUPA5',\n", + " 'HUPA6',\n", + " 'HUPA7',\n", + " 'RP1',\n", + " 'RP2',\n", + " 'RP3',\n", + " 'RP4',\n", + " 'MSA',\n", + " 'ADI',\n", + " 'DMA',\n", + " 'IC1',\n", + " 'IC2',\n", + " 'IC3',\n", + " 'IC4',\n", + " 'IC5',\n", + " 'IC6',\n", + " 'IC7',\n", + " 'IC8',\n", + " 'IC9',\n", + " 'IC10',\n", + " 'IC11',\n", + " 'IC12',\n", + " 'IC13',\n", + " 'IC14',\n", + " 'IC15',\n", + " 'IC16',\n", + " 'IC17',\n", + " 'IC18',\n", + " 'IC19',\n", + " 'IC20',\n", + " 'IC21',\n", + " 'IC22',\n", + " 'IC23',\n", + " 'HHAS1',\n", + " 'HHAS2',\n", + " 'HHAS3',\n", + " 'HHAS4',\n", + " 'MC1',\n", + " 'MC2',\n", + " 'MC3',\n", + " 'TPE1',\n", + " 'TPE2',\n", + " 'TPE3',\n", + " 'TPE4',\n", + " 'TPE5',\n", + " 'TPE6',\n", + " 'TPE7',\n", + " 'TPE8',\n", + " 'TPE9',\n", + " 'PEC1',\n", + " 'PEC2',\n", + " 'TPE10',\n", + " 'TPE11',\n", + " 'TPE12',\n", + " 'TPE13',\n", + " 'LFC1',\n", + " 'LFC2',\n", + " 'LFC3',\n", + " 'LFC4',\n", + " 'LFC5',\n", + " 'LFC6',\n", + " 'LFC7',\n", + " 'LFC8',\n", + " 'LFC9',\n", + " 'LFC10',\n", + " 'OCC1',\n", + " 'OCC2',\n", + " 'OCC3',\n", + " 'OCC4',\n", + " 'OCC5',\n", + " 'OCC6',\n", + " 'OCC7',\n", + " 'OCC8',\n", + " 'OCC9',\n", + " 'OCC10',\n", + " 'OCC11',\n", + " 'OCC12',\n", + " 'OCC13',\n", + " 'EIC1',\n", + " 'EIC2',\n", + " 'EIC3',\n", + " 'EIC4',\n", + " 'EIC5',\n", + " 'EIC6',\n", + " 'EIC7',\n", + " 'EIC8',\n", + " 'EIC9',\n", + " 'EIC10',\n", + " 'EIC11',\n", + " 'EIC12',\n", + " 'EIC13',\n", + " 'EIC14',\n", + " 'EIC15',\n", + " 'EIC16',\n", + " 'OEDC1',\n", + " 'OEDC2',\n", + " 'OEDC3',\n", + " 'OEDC4',\n", + " 'OEDC5',\n", + " 'OEDC6',\n", + " 'OEDC7',\n", + " 'EC1',\n", + " 'EC2',\n", + " 'EC3',\n", + " 'EC4',\n", + " 'EC5',\n", + " 'EC6',\n", + " 'EC7',\n", + " 'EC8',\n", + " 'SEC1',\n", + " 'SEC2',\n", + " 'SEC3',\n", + " 'SEC4',\n", + " 'SEC5',\n", + " 'AFC1',\n", + " 'AFC2',\n", + " 'AFC3',\n", + " 'AFC4',\n", + " 'AFC5',\n", + " 'AFC6',\n", + " 'VC1',\n", + " 'VC2',\n", + " 'VC3',\n", + " 'VC4',\n", + " 'ANC1',\n", + " 'ANC2',\n", + " 'ANC3',\n", + " 'ANC4',\n", + " 'ANC5',\n", + " 'ANC6',\n", + " 'ANC7',\n", + " 'ANC8',\n", + " 'ANC9',\n", + " 'ANC10',\n", + " 'ANC11',\n", + " 'ANC12',\n", + " 'ANC13',\n", + " 'ANC14',\n", + " 'ANC15',\n", + " 'POBC1',\n", + " 'POBC2',\n", + " 'LSC1',\n", + " 'LSC2',\n", + " 'LSC3',\n", + " 'LSC4',\n", + " 'VOC1',\n", + " 'VOC2',\n", + " 'VOC3',\n", + " 'HC1',\n", + " 'HC2',\n", + " 'HC3',\n", + " 'HC4',\n", + " 'HC5',\n", + " 'HC6',\n", + " 'HC7',\n", + " 'HC8',\n", + " 'HC9',\n", + " 'HC10',\n", + " 'HC11',\n", + " 'HC12',\n", + " 'HC13',\n", + " 'HC14',\n", + " 'HC15',\n", + " 'HC16',\n", + " 'HC17',\n", + " 'HC18',\n", + " 'HC19',\n", + " 'HC20',\n", + " 'HC21',\n", + " 'MHUC1',\n", + " 'MHUC2',\n", + " 'AC1',\n", + " 'AC2',\n", + " 'ADATE_2',\n", + " 'ADATE_3',\n", + " 'ADATE_4',\n", + " 'ADATE_5',\n", + " 'ADATE_6',\n", + " 'ADATE_7',\n", + " 'ADATE_8',\n", + " 'ADATE_9',\n", + " 'ADATE_10',\n", + " 'ADATE_11',\n", + " 'ADATE_12',\n", + " 'ADATE_13',\n", + " 'ADATE_14',\n", + " 'ADATE_15',\n", + " 'ADATE_16',\n", + " 'ADATE_17',\n", + " 'ADATE_18',\n", + " 'ADATE_19',\n", + " 'ADATE_20',\n", + " 'ADATE_21',\n", + " 'ADATE_22',\n", + " 'ADATE_23',\n", + " 'ADATE_24',\n", + " 'RFA_2',\n", + " 'RFA_3',\n", + " 'RFA_4',\n", + " 'RFA_5',\n", + " 'RFA_6',\n", + " 'RFA_7',\n", + " 'RFA_8',\n", + " 'RFA_9',\n", + " 'RFA_10',\n", + " 'RFA_11',\n", + " 'RFA_12',\n", + " 'RFA_13',\n", + " 'RFA_14',\n", + " 'RFA_15',\n", + " 'RFA_16',\n", + " 'RFA_17',\n", + " 'RFA_18',\n", + " 'RFA_19',\n", + " 'RFA_20',\n", + " 'RFA_21',\n", + " 'RFA_22',\n", + " 'RFA_23',\n", + " 'RFA_24',\n", + " 'CARDPROM',\n", + " 'MAXADATE',\n", + " 'NUMPROM',\n", + " 'CARDPM12',\n", + " 'NUMPRM12',\n", + " 'RDATE_3',\n", + " 'RDATE_4',\n", + " 'RDATE_5',\n", + " 'RDATE_6',\n", + " 'RDATE_7',\n", + " 'RDATE_8',\n", + " 'RDATE_9',\n", + " 'RDATE_10',\n", + " 'RDATE_11',\n", + " 'RDATE_12',\n", + " 'RDATE_13',\n", + " 'RDATE_14',\n", + " 'RDATE_15',\n", + " 'RDATE_16',\n", + " 'RDATE_17',\n", + " 'RDATE_18',\n", + " 'RDATE_19',\n", + " 'RDATE_20',\n", + " 'RDATE_21',\n", + " 'RDATE_22',\n", + " 'RDATE_23',\n", + " 'RDATE_24',\n", + " 'RAMNT_3',\n", + " 'RAMNT_4',\n", + " 'RAMNT_5',\n", + " 'RAMNT_6',\n", + " 'RAMNT_7',\n", + " 'RAMNT_8',\n", + " 'RAMNT_9',\n", + " 'RAMNT_10',\n", + " 'RAMNT_11',\n", + " 'RAMNT_12',\n", + " 'RAMNT_13',\n", + " 'RAMNT_14',\n", + " 'RAMNT_15',\n", + " 'RAMNT_16',\n", + " 'RAMNT_17',\n", + " 'RAMNT_18',\n", + " 'RAMNT_19',\n", + " 'RAMNT_20',\n", + " 'RAMNT_21',\n", + " 'RAMNT_22',\n", + " 'RAMNT_23',\n", + " 'RAMNT_24',\n", + " 'RAMNTALL',\n", + " 'NGIFTALL',\n", + " 'CARDGIFT',\n", + " 'MINRAMNT',\n", + " 'MINRDATE',\n", + " 'MAXRAMNT',\n", + " 'MAXRDATE',\n", + " 'LASTGIFT',\n", + " 'LASTDATE',\n", + " 'FISTDATE',\n", + " 'NEXTDATE',\n", + " 'TIMELAG',\n", + " 'AVGGIFT',\n", + " 'CONTROLN',\n", + " 'TARGET_B',\n", + " 'TARGET_D',\n", + " 'HPHONE_D',\n", + " 'RFA_2R',\n", + " 'RFA_2F',\n", + " 'RFA_2A',\n", + " 'MDMAUD_R',\n", + " 'MDMAUD_F',\n", + " 'MDMAUD_A',\n", + " 'CLUSTER2',\n", + " 'GEOCODE2']" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(data.columns)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "bdf4bc3e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TARGET_BTARGET_D
000.0
100.0
200.0
300.0
400.0
.........
9540700.0
9540800.0
9540900.0
95410118.0
9541100.0
\n", + "

95412 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " TARGET_B TARGET_D\n", + "0 0 0.0\n", + "1 0 0.0\n", + "2 0 0.0\n", + "3 0 0.0\n", + "4 0 0.0\n", + "... ... ...\n", + "95407 0 0.0\n", + "95408 0 0.0\n", + "95409 0 0.0\n", + "95410 1 18.0\n", + "95411 0 0.0\n", + "\n", + "[95412 rows x 2 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data[['TARGET_B','TARGET_D']]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "c9fdcea8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ODATEDWTCODEDOBAGENUMCHLDINCOMEWEALTH1HITMBCRAFTMBGARDEN...FISTDATENEXTDATETIMELAGAVGGIFTCONTROLNTARGET_BTARGET_DHPHONE_DRFA_2FCLUSTER2
count95412.00000095412.00000095412.00000071747.00000012386.00000074126.00000050680.00000095412.00000042558.00000042558.000000...95412.00000085439.00000085439.00000095412.00000095412.00000095412.00000095412.00000095412.00000095412.00000095280.000000
mean9141.36325654.2231172723.60293361.6116491.5277733.8862485.3456993.3214380.1520750.059166...9135.6516489151.0229178.09373913.34778695778.1769590.0507590.7930730.5006181.91005331.533711
std343.454752953.8444762132.24129516.6641990.8068611.8549602.7424909.3068990.4700230.262078...320.394019294.2572608.21324210.76999755284.5960940.2195064.4297250.5000021.07274918.764614
min8306.0000000.0000000.0000001.0000001.0000001.0000000.0000000.0000000.0000000.000000...0.0000007211.0000000.0000001.2857141.0000000.0000000.0000000.0000001.0000001.000000
25%8801.0000000.000000201.00000048.0000001.0000002.0000003.0000000.0000000.0000000.000000...8810.0000008903.0000004.0000008.38461547910.7500000.0000000.0000000.0000001.00000015.000000
50%9201.0000001.0000002610.00000062.0000001.0000004.0000006.0000000.0000000.0000000.000000...9201.0000009204.0000006.00000011.63636495681.5000000.0000000.0000001.0000002.00000032.000000
75%9501.0000002.0000004601.00000075.0000002.0000005.0000008.0000003.0000000.0000000.000000...9409.0000009409.00000011.00000015.477955143643.5000000.0000000.0000001.0000003.00000049.000000
max9701.00000072002.0000009710.00000098.0000007.0000007.0000009.000000241.0000006.0000004.000000...9603.0000009702.0000001088.0000001000.000000191779.0000001.000000200.0000001.0000004.00000062.000000
\n", + "

8 rows × 407 columns

\n", + "
" + ], + "text/plain": [ + " ODATEDW TCODE DOB AGE NUMCHLD \\\n", + "count 95412.000000 95412.000000 95412.000000 71747.000000 12386.000000 \n", + "mean 9141.363256 54.223117 2723.602933 61.611649 1.527773 \n", + "std 343.454752 953.844476 2132.241295 16.664199 0.806861 \n", + "min 8306.000000 0.000000 0.000000 1.000000 1.000000 \n", + "25% 8801.000000 0.000000 201.000000 48.000000 1.000000 \n", + "50% 9201.000000 1.000000 2610.000000 62.000000 1.000000 \n", + "75% 9501.000000 2.000000 4601.000000 75.000000 2.000000 \n", + "max 9701.000000 72002.000000 9710.000000 98.000000 7.000000 \n", + "\n", + " INCOME WEALTH1 HIT MBCRAFT MBGARDEN \\\n", + "count 74126.000000 50680.000000 95412.000000 42558.000000 42558.000000 \n", + "mean 3.886248 5.345699 3.321438 0.152075 0.059166 \n", + "std 1.854960 2.742490 9.306899 0.470023 0.262078 \n", + "min 1.000000 0.000000 0.000000 0.000000 0.000000 \n", + "25% 2.000000 3.000000 0.000000 0.000000 0.000000 \n", + "50% 4.000000 6.000000 0.000000 0.000000 0.000000 \n", + "75% 5.000000 8.000000 3.000000 0.000000 0.000000 \n", + "max 7.000000 9.000000 241.000000 6.000000 4.000000 \n", + "\n", + " ... FISTDATE NEXTDATE TIMELAG AVGGIFT \\\n", + "count ... 95412.000000 85439.000000 85439.000000 95412.000000 \n", + "mean ... 9135.651648 9151.022917 8.093739 13.347786 \n", + "std ... 320.394019 294.257260 8.213242 10.769997 \n", + "min ... 0.000000 7211.000000 0.000000 1.285714 \n", + "25% ... 8810.000000 8903.000000 4.000000 8.384615 \n", + "50% ... 9201.000000 9204.000000 6.000000 11.636364 \n", + "75% ... 9409.000000 9409.000000 11.000000 15.477955 \n", + "max ... 9603.000000 9702.000000 1088.000000 1000.000000 \n", + "\n", + " CONTROLN TARGET_B TARGET_D HPHONE_D RFA_2F \\\n", + "count 95412.000000 95412.000000 95412.000000 95412.000000 95412.000000 \n", + "mean 95778.176959 0.050759 0.793073 0.500618 1.910053 \n", + "std 55284.596094 0.219506 4.429725 0.500002 1.072749 \n", + "min 1.000000 0.000000 0.000000 0.000000 1.000000 \n", + "25% 47910.750000 0.000000 0.000000 0.000000 1.000000 \n", + "50% 95681.500000 0.000000 0.000000 1.000000 2.000000 \n", + "75% 143643.500000 0.000000 0.000000 1.000000 3.000000 \n", + "max 191779.000000 1.000000 200.000000 1.000000 4.000000 \n", + "\n", + " CLUSTER2 \n", + "count 95280.000000 \n", + "mean 31.533711 \n", + "std 18.764614 \n", + "min 1.000000 \n", + "25% 15.000000 \n", + "50% 32.000000 \n", + "75% 49.000000 \n", + "max 62.000000 \n", + "\n", + "[8 rows x 407 columns]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.describe()" + ] + }, + { + "cell_type": "markdown", + "id": "88623446", + "metadata": {}, + "source": [ + "Data imbalance detected between people who did not donate and people who donated, speacially if we are talking about the high amount donors:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0abebfa7", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(950, 46)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(data[data['TARGET_D'] > 20]['TARGET_D']), len(data[data['TARGET_D'] > 50]['TARGET_D'])" + ] + }, + { + "cell_type": "markdown", + "id": "109da0ed", + "metadata": {}, + "source": [ + "Here we notice there are no highly correlated columns, which is likely to reflect in a bad fit for linear regression:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "42fe3505", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "TARGET_D 1.000000\n", + "TARGET_B 0.774232\n", + "RAMNT_4 0.268811\n", + "RAMNT_21 0.099339\n", + "RAMNT_9 0.090168\n", + " ... \n", + "RDATE_3 -0.125194\n", + "RDATE_5 -0.220455\n", + "RAMNT_5 -0.272147\n", + "ADATE_5 NaN\n", + "ADATE_15 NaN\n", + "Name: TARGET_D, Length: 407, dtype: float64" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.corr()['TARGET_D'].sort_values(ascending=False)" + ] + }, { "cell_type": "markdown", "id": "5bbb5252", @@ -26,11 +1101,110 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "dbe69e8b", "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
percentage
RDATE_599.990567
RAMNT_599.990567
RDATE_399.746363
RAMNT_399.746363
RDATE_499.705488
......
ADATE_1621.343227
ADATE_1419.774242
ADATE_911.785729
ADATE_1110.923154
NEXTDATE10.452564
\n", + "

80 rows × 1 columns

\n", + "
" + ], + "text/plain": [ + " percentage\n", + "RDATE_5 99.990567\n", + "RAMNT_5 99.990567\n", + "RDATE_3 99.746363\n", + "RAMNT_3 99.746363\n", + "RDATE_4 99.705488\n", + "... ...\n", + "ADATE_16 21.343227\n", + "ADATE_14 19.774242\n", + "ADATE_9 11.785729\n", + "ADATE_11 10.923154\n", + "NEXTDATE 10.452564\n", + "\n", + "[80 rows x 1 columns]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nulls = pd.DataFrame(data.isna().sum()*100/len(data), columns=['percentage'])\n", + "nulls.sort_values('percentage', ascending = False).head(80)" + ] }, { "cell_type": "markdown", @@ -45,11 +1219,63 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "3a07b75a", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "drop_list = ['OSOURCE', 'ZIP', 'MAILCODE', 'PVASTATE']\n", + "# as there is no 'ZIP CODE' column, I assumed that should be the 'ZIP' column, but we can also add columns 'MAILCODE' and 'PVASTATE'" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "5846dae9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MBC 4539\n", + "SYN 3563\n", + "AML 3430\n", + "BHG 3324\n", + "IMP 2986\n", + " ... \n", + "BAR 1\n", + "BCN 1\n", + "AMS 1\n", + "BUR 1\n", + "SIF 1\n", + "Name: OSOURCE, Length: 896, dtype: int64\n", + "85351 61\n", + "92653 59\n", + "85710 54\n", + "95608 50\n", + "60619 45\n", + " ..\n", + "67104- 1\n", + "28438 1\n", + "76885 1\n", + "55364- 1\n", + "33602 1\n", + "Name: ZIP, Length: 19938, dtype: int64\n", + " 94013\n", + "B 1399\n", + "Name: MAILCODE, dtype: int64\n", + " 93954\n", + "P 1453\n", + "E 5\n", + "Name: PVASTATE, dtype: int64\n" + ] + } + ], + "source": [ + "for x in drop_list:\n", + " print(data[x].value_counts())" + ] }, { "cell_type": "markdown", @@ -61,11 +1287,176 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "2b6f01b6", "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
percentage
NUMCHLD87.018404
RDATE_399.746363
RDATE_499.705488
RDATE_599.990567
RDATE_699.186685
RDATE_790.677273
RDATE_1089.035970
RDATE_1387.160944
RDATE_1592.388798
RDATE_1790.146942
RDATE_2091.732696
RDATE_2190.029556
RDATE_2391.763091
RAMNT_399.746363
RAMNT_499.705488
RAMNT_599.990567
RAMNT_699.186685
RAMNT_790.677273
RAMNT_1089.035970
RAMNT_1387.160944
RAMNT_1592.388798
RAMNT_1790.146942
RAMNT_2091.732696
RAMNT_2190.029556
RAMNT_2391.763091
\n", + "
" + ], + "text/plain": [ + " percentage\n", + "NUMCHLD 87.018404\n", + "RDATE_3 99.746363\n", + "RDATE_4 99.705488\n", + "RDATE_5 99.990567\n", + "RDATE_6 99.186685\n", + "RDATE_7 90.677273\n", + "RDATE_10 89.035970\n", + "RDATE_13 87.160944\n", + "RDATE_15 92.388798\n", + "RDATE_17 90.146942\n", + "RDATE_20 91.732696\n", + "RDATE_21 90.029556\n", + "RDATE_23 91.763091\n", + "RAMNT_3 99.746363\n", + "RAMNT_4 99.705488\n", + "RAMNT_5 99.990567\n", + "RAMNT_6 99.186685\n", + "RAMNT_7 90.677273\n", + "RAMNT_10 89.035970\n", + "RAMNT_13 87.160944\n", + "RAMNT_15 92.388798\n", + "RAMNT_17 90.146942\n", + "RAMNT_20 91.732696\n", + "RAMNT_21 90.029556\n", + "RAMNT_23 91.763091" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nulls[nulls['percentage'] > 85]" + ] }, { "cell_type": "markdown", @@ -77,11 +1468,386 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "ca6c4d91", + "execution_count": 25, + "id": "5a0645e0", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "nulls_85 = list(nulls[nulls['percentage'] > 85].index)\n", + "drop_list.extend(nulls_85)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "ca6c4d91", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ODATEDWTCODESTATEDOBNOEXCHRECINHSERECP3RECPGVGRECSWEEPMDMAUD...TARGET_DHPHONE_DRFA_2RRFA_2FRFA_2AMDMAUD_RMDMAUD_FMDMAUD_ACLUSTER2GEOCODE2
089010IL37120XXXX...0.00L4EXXX39.0C
194011CA52020XXXX...0.00L2GXXX1.0A
290011NC00XXXX...0.01L4EXXX60.0C
387010CA28010XXXX...0.01L4EXXX41.0C
486010FL20010XXXXXX...0.01L2FXXX26.0A
..................................................................
9540796011AK00XXXX...0.00L1GXXX12.0C
9540896011TX50010XXXX...0.01L1FXXX2.0A
9540995011MI38010XXXXX...0.01L3EXXX34.0B
9541086010CA40050XXXXX...18.01L4FXXX11.0A
9541188012NC18010XXC1CM...0.01L1GC1C12.0C
\n", + "

95412 rows × 452 columns

\n", + "
" + ], + "text/plain": [ + " ODATEDW TCODE STATE DOB NOEXCH RECINHSE RECP3 RECPGVG RECSWEEP \\\n", + "0 8901 0 IL 3712 0 \n", + "1 9401 1 CA 5202 0 \n", + "2 9001 1 NC 0 0 \n", + "3 8701 0 CA 2801 0 \n", + "4 8601 0 FL 2001 0 X X \n", + "... ... ... ... ... ... ... ... ... ... \n", + "95407 9601 1 AK 0 0 \n", + "95408 9601 1 TX 5001 0 \n", + "95409 9501 1 MI 3801 0 X \n", + "95410 8601 0 CA 4005 0 X \n", + "95411 8801 2 NC 1801 0 X X \n", + "\n", + " MDMAUD ... TARGET_D HPHONE_D RFA_2R RFA_2F RFA_2A MDMAUD_R MDMAUD_F \\\n", + "0 XXXX ... 0.0 0 L 4 E X X \n", + "1 XXXX ... 0.0 0 L 2 G X X \n", + "2 XXXX ... 0.0 1 L 4 E X X \n", + "3 XXXX ... 0.0 1 L 4 E X X \n", + "4 XXXX ... 0.0 1 L 2 F X X \n", + "... ... ... ... ... ... ... ... ... ... \n", + "95407 XXXX ... 0.0 0 L 1 G X X \n", + "95408 XXXX ... 0.0 1 L 1 F X X \n", + "95409 XXXX ... 0.0 1 L 3 E X X \n", + "95410 XXXX ... 18.0 1 L 4 F X X \n", + "95411 C1CM ... 0.0 1 L 1 G C 1 \n", + "\n", + " MDMAUD_A CLUSTER2 GEOCODE2 \n", + "0 X 39.0 C \n", + "1 X 1.0 A \n", + "2 X 60.0 C \n", + "3 X 41.0 C \n", + "4 X 26.0 A \n", + "... ... ... ... \n", + "95407 X 12.0 C \n", + "95408 X 2.0 A \n", + "95409 X 34.0 B \n", + "95410 X 11.0 A \n", + "95411 C 12.0 C \n", + "\n", + "[95412 rows x 452 columns]" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.drop(columns = drop_list)" + ] }, { "cell_type": "markdown", @@ -100,11 +1866,519 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "b583feaa", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "F 51277\n", + "M 39094\n", + " 2957\n", + "U 1715\n", + "J 365\n", + "A 2\n", + "C 2\n", + "Name: GENDER, dtype: int64\n" + ] + } + ], + "source": [ + "print(data['GENDER'].value_counts())" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "c6a6a668", + "metadata": {}, + "outputs": [], + "source": [ + "data['GENDER'] = data['GENDER'].fillna('F')" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "221b31bd", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "F 51277\n", + "M 39094\n", + " 2957\n", + "U 1715\n", + "J 365\n", + "A 2\n", + "C 2\n", + "Name: GENDER, dtype: int64\n" + ] + } + ], + "source": [ + "print(data['GENDER'].value_counts())" + ] + }, + { + "cell_type": "markdown", + "id": "0db76108", + "metadata": {}, + "source": [ + "- As nothing happened I'll check the empty spaces ' '" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "e72bb181", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "data['GENDER'] = data['GENDER'].replace(' ', 'F')" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "01e6d084", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "F 54234\n", + "M 39094\n", + "U 1715\n", + "J 365\n", + "A 2\n", + "C 2\n", + "Name: GENDER, dtype: int64\n" + ] + } + ], + "source": [ + "print(data['GENDER'].value_counts())" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "id": "42b06823", + "metadata": {}, + "outputs": [], + "source": [ + "def clean_gender(data, column):\n", + " gender = ['F', 'M']\n", + " genders_to_clean = [i for i in data['GENDER'].unique() if i not in gender]\n", + " data['GENDER'] = data['GENDER'].replace(genders_to_clean, 'other')\n", + " return data" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "id": "d601220d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ODATEDWOSOURCETCODESTATEZIPMAILCODEPVASTATEDOBNOEXCHRECINHSE...TARGET_DHPHONE_DRFA_2RRFA_2FRFA_2AMDMAUD_RMDMAUD_FMDMAUD_ACLUSTER2GEOCODE2
08901GRI0IL6108137120...0.00L4EXXX39.0C
19401BOA1CA9132652020...0.00L2GXXX1.0A
29001AMH1NC2701700...0.01L4EXXX60.0C
38701BRY0CA9595328010...0.01L4EXXX41.0C
486010FL3317620010X...0.01L2FXXX26.0A
..................................................................
954079601ASE1AK9950400...0.00L1GXXX12.0C
954089601DCD1TX7737950010...0.01L1FXXX2.0A
954099501MBC1MI4891038010...0.01L3EXXX34.0B
954108601PRV0CA9132040050X...18.01L4FXXX11.0A
954118801MCC2NC2840918010X...0.01L1GC1C12.0C
\n", + "

95412 rows × 481 columns

\n", + "
" + ], + "text/plain": [ + " ODATEDW OSOURCE TCODE STATE ZIP MAILCODE PVASTATE DOB NOEXCH \\\n", + "0 8901 GRI 0 IL 61081 3712 0 \n", + "1 9401 BOA 1 CA 91326 5202 0 \n", + "2 9001 AMH 1 NC 27017 0 0 \n", + "3 8701 BRY 0 CA 95953 2801 0 \n", + "4 8601 0 FL 33176 2001 0 \n", + "... ... ... ... ... ... ... ... ... ... \n", + "95407 9601 ASE 1 AK 99504 0 0 \n", + "95408 9601 DCD 1 TX 77379 5001 0 \n", + "95409 9501 MBC 1 MI 48910 3801 0 \n", + "95410 8601 PRV 0 CA 91320 4005 0 \n", + "95411 8801 MCC 2 NC 28409 1801 0 \n", + "\n", + " RECINHSE ... TARGET_D HPHONE_D RFA_2R RFA_2F RFA_2A MDMAUD_R MDMAUD_F \\\n", + "0 ... 0.0 0 L 4 E X X \n", + "1 ... 0.0 0 L 2 G X X \n", + "2 ... 0.0 1 L 4 E X X \n", + "3 ... 0.0 1 L 4 E X X \n", + "4 X ... 0.0 1 L 2 F X X \n", + "... ... ... ... ... ... ... ... ... ... \n", + "95407 ... 0.0 0 L 1 G X X \n", + "95408 ... 0.0 1 L 1 F X X \n", + "95409 ... 0.0 1 L 3 E X X \n", + "95410 X ... 18.0 1 L 4 F X X \n", + "95411 X ... 0.0 1 L 1 G C 1 \n", + "\n", + " MDMAUD_A CLUSTER2 GEOCODE2 \n", + "0 X 39.0 C \n", + "1 X 1.0 A \n", + "2 X 60.0 C \n", + "3 X 41.0 C \n", + "4 X 26.0 A \n", + "... ... ... ... \n", + "95407 X 12.0 C \n", + "95408 X 2.0 A \n", + "95409 X 34.0 B \n", + "95410 X 11.0 A \n", + "95411 C 12.0 C \n", + "\n", + "[95412 rows x 481 columns]" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "clean_gender(data, data['GENDER'])" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "id": "e181167f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "F 54234\n", + "M 39094\n", + "other 2084\n", + "Name: GENDER, dtype: int64" + ] + }, + "execution_count": 70, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data['GENDER'].value_counts()" + ] } ], "metadata": { @@ -123,7 +2397,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.8.8" } }, "nbformat": 4,