Hesoyam-2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# --Training--"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>pisat</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>...</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "      <th>kelasrawat</th>\n",
       "      <th>kdsa</th>\n",
       "      <th>kdsp</th>\n",
       "      <th>kdsr</th>\n",
       "      <th>kdsi</th>\n",
       "      <th>kdsd</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-42-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-05-27T17:00:00.000Z</td>\n",
       "      <td>2019-05-30T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>D-4-13-III</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401877</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>301</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401878</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>233</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-13-II</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401879</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>118</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-06T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401880</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>101</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401881</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>12</td>\n",
       "      <td>SD</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-16-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11401882 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  dati2 typefaskes  usia jenkel  pisat  \\\n",
       "0           165666         486     17         KL    48      P    1.0   \n",
       "1          1010828         520     17          A    63      L    1.0   \n",
       "2           166042         523     17         KL    53      P    1.0   \n",
       "3           168937         549     17         KL    54      P    1.0   \n",
       "4          1005899         549     17          A    53      P    1.0   \n",
       "...            ...         ...    ...        ...   ...    ...    ...   \n",
       "11401877   9983563     8527704    301          B     0      P    4.0   \n",
       "11401878  11053870     8527709    233          A     0      P    4.0   \n",
       "11401879   7461049     8527751    118          B     0      L    4.0   \n",
       "11401880   1075162     8527853    101         SC     0      L    5.0   \n",
       "11401881    102794     8527876     12         SD     0      P    4.0   \n",
       "\n",
       "                         tgldatang                 tglpulang  jenispel  ...  \\\n",
       "0         2018-07-25T17:00:00.000Z  2018-07-25T17:00:00.000Z         2  ...   \n",
       "1         2019-05-27T17:00:00.000Z  2019-05-30T17:00:00.000Z         1  ...   \n",
       "2         2019-07-16T17:00:00.000Z  2019-07-16T17:00:00.000Z         2  ...   \n",
       "3         2019-10-17T17:00:00.000Z  2019-10-17T17:00:00.000Z         2  ...   \n",
       "4         2018-04-18T17:00:00.000Z  2018-04-18T17:00:00.000Z         2  ...   \n",
       "...                            ...                       ...       ...  ...   \n",
       "11401877  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1  ...   \n",
       "11401878  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1  ...   \n",
       "11401879  2020-12-30T17:00:00.000Z  2021-01-06T17:00:00.000Z         1  ...   \n",
       "11401880  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1  ...   \n",
       "11401881  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1  ...   \n",
       "\n",
       "               biaya jenispulang         cbg  kelasrawat  kdsa  kdsp  kdsr  \\\n",
       "0           184300.0         1.0    Q-5-42-0           3  None  None  None   \n",
       "1         10628400.0         1.0  D-4-13-III           1  None  None  None   \n",
       "2           187300.0         1.0    Q-5-44-0           3  None  None  None   \n",
       "3           187300.0         1.0    Q-5-44-0           3  None  None  None   \n",
       "4           381600.0         1.0    Q-5-44-0           3  None  None  None   \n",
       "...              ...         ...         ...         ...   ...   ...   ...   \n",
       "11401877   3645100.0         1.0    P-8-17-I           3  None  None  None   \n",
       "11401878  12273200.0         1.0   P-8-13-II           2  None  None  None   \n",
       "11401879   4099000.0         1.0    P-8-08-I           3  None  None  None   \n",
       "11401880   3992400.0         1.0    P-8-08-I           3  None  None  None   \n",
       "11401881   1624900.0         1.0    P-8-16-I           3  None  None  None   \n",
       "\n",
       "          kdsi  kdsd label  \n",
       "0         None  None     0  \n",
       "1         None  None     0  \n",
       "2         None  None     0  \n",
       "3         None  None     0  \n",
       "4         None  None     0  \n",
       "...        ...   ...   ...  \n",
       "11401877  None  None     0  \n",
       "11401878  None  None     0  \n",
       "11401879  None  None     0  \n",
       "11401880  None  None     0  \n",
       "11401881  None  None     0  \n",
       "\n",
       "[11401882 rows x 22 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading sampling\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "samp = pd.read_csv(\"Dataset/sampling_healtkathon2022.csv\")\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>69.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202866</th>\n",
       "      <td>11426915</td>\n",
       "      <td>93.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202867</th>\n",
       "      <td>11426927</td>\n",
       "      <td>89.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202868</th>\n",
       "      <td>11426927</td>\n",
       "      <td>96.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202869</th>\n",
       "      <td>11426927</td>\n",
       "      <td>87.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202870</th>\n",
       "      <td>11426927</td>\n",
       "      <td>93.96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12202871 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id   proc\n",
       "0                6  90.59\n",
       "1                6  69.01\n",
       "2                6  88.78\n",
       "3               27  90.59\n",
       "4               27  88.78\n",
       "...            ...    ...\n",
       "12202866  11426915  93.57\n",
       "12202867  11426927  89.52\n",
       "12202868  11426927  96.55\n",
       "12202869  11426927  87.39\n",
       "12202870  11426927  93.96\n",
       "\n",
       "[12202871 rows x 2 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading proc\n",
    "proc = pd.read_csv(\"dataset/sampling_healthkathon2022_procedure.csv\")\n",
    "proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>69.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202866</th>\n",
       "      <td>11426915</td>\n",
       "      <td>93.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202867</th>\n",
       "      <td>11426927</td>\n",
       "      <td>89.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202868</th>\n",
       "      <td>11426927</td>\n",
       "      <td>96.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202869</th>\n",
       "      <td>11426927</td>\n",
       "      <td>87.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202870</th>\n",
       "      <td>11426927</td>\n",
       "      <td>93.96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12202871 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id   proc\n",
       "0                6  90.59\n",
       "1                6  69.01\n",
       "2                6  88.78\n",
       "3               27  90.59\n",
       "4               27  88.78\n",
       "...            ...    ...\n",
       "12202866  11426915  93.57\n",
       "12202867  11426927  89.52\n",
       "12202868  11426927  96.55\n",
       "12202869  11426927  87.39\n",
       "12202870  11426927  93.96\n",
       "\n",
       "[12202871 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading diag\n",
    "diag = pd.read_csv(\"dataset/sampling_healthkathon2022_procedure.csv\")\n",
    "diag"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleansing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>pisat</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>...</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "      <th>kelasrawat</th>\n",
       "      <th>kdsa</th>\n",
       "      <th>kdsp</th>\n",
       "      <th>kdsr</th>\n",
       "      <th>kdsi</th>\n",
       "      <th>kdsd</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-42-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-05-27T17:00:00.000Z</td>\n",
       "      <td>2019-05-30T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>D-4-13-III</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401877</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>301</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401878</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>233</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-13-II</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401879</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>118</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-06T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401880</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>101</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401881</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>12</td>\n",
       "      <td>SD</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-16-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11401882 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  dati2 typefaskes  usia jenkel  pisat  \\\n",
       "0           165666         486     17         KL    48      P    1.0   \n",
       "1          1010828         520     17          A    63      L    1.0   \n",
       "2           166042         523     17         KL    53      P    1.0   \n",
       "3           168937         549     17         KL    54      P    1.0   \n",
       "4          1005899         549     17          A    53      P    1.0   \n",
       "...            ...         ...    ...        ...   ...    ...    ...   \n",
       "11401877   9983563     8527704    301          B     0      P    4.0   \n",
       "11401878  11053870     8527709    233          A     0      P    4.0   \n",
       "11401879   7461049     8527751    118          B     0      L    4.0   \n",
       "11401880   1075162     8527853    101         SC     0      L    5.0   \n",
       "11401881    102794     8527876     12         SD     0      P    4.0   \n",
       "\n",
       "                         tgldatang                 tglpulang  jenispel  ...  \\\n",
       "0         2018-07-25T17:00:00.000Z  2018-07-25T17:00:00.000Z         2  ...   \n",
       "1         2019-05-27T17:00:00.000Z  2019-05-30T17:00:00.000Z         1  ...   \n",
       "2         2019-07-16T17:00:00.000Z  2019-07-16T17:00:00.000Z         2  ...   \n",
       "3         2019-10-17T17:00:00.000Z  2019-10-17T17:00:00.000Z         2  ...   \n",
       "4         2018-04-18T17:00:00.000Z  2018-04-18T17:00:00.000Z         2  ...   \n",
       "...                            ...                       ...       ...  ...   \n",
       "11401877  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1  ...   \n",
       "11401878  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1  ...   \n",
       "11401879  2020-12-30T17:00:00.000Z  2021-01-06T17:00:00.000Z         1  ...   \n",
       "11401880  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1  ...   \n",
       "11401881  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1  ...   \n",
       "\n",
       "               biaya jenispulang         cbg  kelasrawat  kdsa  kdsp  kdsr  \\\n",
       "0           184300.0         1.0    Q-5-42-0           3  None  None  None   \n",
       "1         10628400.0         1.0  D-4-13-III           1  None  None  None   \n",
       "2           187300.0         1.0    Q-5-44-0           3  None  None  None   \n",
       "3           187300.0         1.0    Q-5-44-0           3  None  None  None   \n",
       "4           381600.0         1.0    Q-5-44-0           3  None  None  None   \n",
       "...              ...         ...         ...         ...   ...   ...   ...   \n",
       "11401877   3645100.0         1.0    P-8-17-I           3  None  None  None   \n",
       "11401878  12273200.0         1.0   P-8-13-II           2  None  None  None   \n",
       "11401879   4099000.0         1.0    P-8-08-I           3  None  None  None   \n",
       "11401880   3992400.0         1.0    P-8-08-I           3  None  None  None   \n",
       "11401881   1624900.0         1.0    P-8-16-I           3  None  None  None   \n",
       "\n",
       "          kdsi  kdsd label  \n",
       "0         None  None     0  \n",
       "1         None  None     0  \n",
       "2         None  None     0  \n",
       "3         None  None     0  \n",
       "4         None  None     0  \n",
       "...        ...   ...   ...  \n",
       "11401877  None  None     0  \n",
       "11401878  None  None     0  \n",
       "11401879  None  None     0  \n",
       "11401880  None  None     0  \n",
       "11401881  None  None     0  \n",
       "\n",
       "[11401882 rows x 22 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading samp\n",
    "import pandas as pd\n",
    "import gc\n",
    "samp = pd.read_csv(\"dataset/sampling_healtkathon2022.csv\")\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>pisat</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>...</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "      <th>kelasrawat</th>\n",
       "      <th>kdsa</th>\n",
       "      <th>kdsp</th>\n",
       "      <th>kdsr</th>\n",
       "      <th>kdsi</th>\n",
       "      <th>kdsd</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-42-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-05-27T17:00:00.000Z</td>\n",
       "      <td>2019-05-30T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>D-4-13-III</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401877</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>301</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401878</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>233</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-13-II</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401879</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>118</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-06T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401880</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>101</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401881</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>12</td>\n",
       "      <td>SD</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-16-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11401868 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  dati2 typefaskes  usia jenkel  pisat  \\\n",
       "0           165666         486     17         KL    48      P    1.0   \n",
       "1          1010828         520     17          A    63      L    1.0   \n",
       "2           166042         523     17         KL    53      P    1.0   \n",
       "3           168937         549     17         KL    54      P    1.0   \n",
       "4          1005899         549     17          A    53      P    1.0   \n",
       "...            ...         ...    ...        ...   ...    ...    ...   \n",
       "11401877   9983563     8527704    301          B     0      P    4.0   \n",
       "11401878  11053870     8527709    233          A     0      P    4.0   \n",
       "11401879   7461049     8527751    118          B     0      L    4.0   \n",
       "11401880   1075162     8527853    101         SC     0      L    5.0   \n",
       "11401881    102794     8527876     12         SD     0      P    4.0   \n",
       "\n",
       "                         tgldatang                 tglpulang  jenispel  ...  \\\n",
       "0         2018-07-25T17:00:00.000Z  2018-07-25T17:00:00.000Z         2  ...   \n",
       "1         2019-05-27T17:00:00.000Z  2019-05-30T17:00:00.000Z         1  ...   \n",
       "2         2019-07-16T17:00:00.000Z  2019-07-16T17:00:00.000Z         2  ...   \n",
       "3         2019-10-17T17:00:00.000Z  2019-10-17T17:00:00.000Z         2  ...   \n",
       "4         2018-04-18T17:00:00.000Z  2018-04-18T17:00:00.000Z         2  ...   \n",
       "...                            ...                       ...       ...  ...   \n",
       "11401877  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1  ...   \n",
       "11401878  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1  ...   \n",
       "11401879  2020-12-30T17:00:00.000Z  2021-01-06T17:00:00.000Z         1  ...   \n",
       "11401880  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1  ...   \n",
       "11401881  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1  ...   \n",
       "\n",
       "               biaya jenispulang         cbg  kelasrawat  kdsa  kdsp  kdsr  \\\n",
       "0           184300.0         1.0    Q-5-42-0           3  None  None  None   \n",
       "1         10628400.0         1.0  D-4-13-III           1  None  None  None   \n",
       "2           187300.0         1.0    Q-5-44-0           3  None  None  None   \n",
       "3           187300.0         1.0    Q-5-44-0           3  None  None  None   \n",
       "4           381600.0         1.0    Q-5-44-0           3  None  None  None   \n",
       "...              ...         ...         ...         ...   ...   ...   ...   \n",
       "11401877   3645100.0         1.0    P-8-17-I           3  None  None  None   \n",
       "11401878  12273200.0         1.0   P-8-13-II           2  None  None  None   \n",
       "11401879   4099000.0         1.0    P-8-08-I           3  None  None  None   \n",
       "11401880   3992400.0         1.0    P-8-08-I           3  None  None  None   \n",
       "11401881   1624900.0         1.0    P-8-16-I           3  None  None  None   \n",
       "\n",
       "          kdsi  kdsd label  \n",
       "0         None  None     0  \n",
       "1         None  None     0  \n",
       "2         None  None     0  \n",
       "3         None  None     0  \n",
       "4         None  None     0  \n",
       "...        ...   ...   ...  \n",
       "11401877  None  None     0  \n",
       "11401878  None  None     0  \n",
       "11401879  None  None     0  \n",
       "11401880  None  None     0  \n",
       "11401881  None  None     0  \n",
       "\n",
       "[11401868 rows x 22 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dropping duplicate id because it must be unique\n",
    "samp.drop_duplicates(subset=\"id\", inplace=True)\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                   0\n",
       "id_peserta           0\n",
       "dati2                0\n",
       "typefaskes           0\n",
       "usia                 0\n",
       "jenkel              49\n",
       "pisat              190\n",
       "tgldatang            0\n",
       "tglpulang            0\n",
       "jenispel             0\n",
       "politujuan     4041444\n",
       "diagfktp          2530\n",
       "biaya            57815\n",
       "jenispulang         39\n",
       "cbg                  0\n",
       "kelasrawat           0\n",
       "kdsa            198670\n",
       "kdsp            197007\n",
       "kdsr            198459\n",
       "kdsi            198463\n",
       "kdsd            195181\n",
       "label                0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if there is any null value\n",
    "samp.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dati2: 489 unique\n",
      "typefaskes: 26 unique\n",
      "usia: 111 unique\n",
      "jenkel: 3 unique\n",
      "pisat: 6 unique\n",
      "tgldatang: 1096 unique\n",
      "tglpulang: 1247 unique\n",
      "jenispel: 2 unique\n",
      "biaya: 57855 unique\n",
      "jenispulang: 6 unique\n",
      "cbg: 1034 unique\n",
      "kelasrawat: 3 unique\n"
     ]
    }
   ],
   "source": [
    "# check the amount of unique value each column\n",
    "for i in samp.columns:\n",
    "    if i not in [\"id\", \"id_peserta\", \"diagfktp\", \"label\", \"kdsa\", \"kdsp\", \"kdsr\", \"kdsi\", \"kdsd\", \"politujuan\"]:\n",
    "        print(f\"{i}: {len(samp[i].unique())} unique\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "id_peserta     0\n",
       "dati2          0\n",
       "typefaskes     0\n",
       "usia           0\n",
       "jenkel         0\n",
       "pisat          0\n",
       "tgldatang      0\n",
       "tglpulang      0\n",
       "jenispel       0\n",
       "biaya          0\n",
       "jenispulang    0\n",
       "cbg            0\n",
       "kelasrawat     0\n",
       "label          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# droping too many unique value and in the kdsa, kdsp, ... there is many null value \n",
    "# but it filled with string \"None\"\n",
    "samp.drop([\"politujuan\", \"diagfktp\", \"kdsa\", \"kdsp\", \"kdsr\", \"kdsi\", \"kdsd\"], axis=1, inplace=True)\n",
    "samp[\"biaya\"].fillna(0, inplace=True)\n",
    "samp.dropna(axis=0, inplace=True)\n",
    "samp.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>pisat</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "      <th>kelasrawat</th>\n",
       "      <th>label</th>\n",
       "      <th>zs_biaya</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-42-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.375566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-05-27T17:00:00.000Z</td>\n",
       "      <td>2019-05-30T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>D-4-13-III</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.634214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.374989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.374989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.337599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401877</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>301</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.290403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401878</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>233</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-13-II</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.950726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401879</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>118</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-06T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.377748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401880</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>101</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.357234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401881</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>12</td>\n",
       "      <td>SD</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-16-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.098349</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11401590 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  dati2 typefaskes  usia jenkel  pisat  \\\n",
       "0           165666         486     17         KL    48      P    1.0   \n",
       "1          1010828         520     17          A    63      L    1.0   \n",
       "2           166042         523     17         KL    53      P    1.0   \n",
       "3           168937         549     17         KL    54      P    1.0   \n",
       "4          1005899         549     17          A    53      P    1.0   \n",
       "...            ...         ...    ...        ...   ...    ...    ...   \n",
       "11401877   9983563     8527704    301          B     0      P    4.0   \n",
       "11401878  11053870     8527709    233          A     0      P    4.0   \n",
       "11401879   7461049     8527751    118          B     0      L    4.0   \n",
       "11401880   1075162     8527853    101         SC     0      L    5.0   \n",
       "11401881    102794     8527876     12         SD     0      P    4.0   \n",
       "\n",
       "                         tgldatang                 tglpulang  jenispel  \\\n",
       "0         2018-07-25T17:00:00.000Z  2018-07-25T17:00:00.000Z         2   \n",
       "1         2019-05-27T17:00:00.000Z  2019-05-30T17:00:00.000Z         1   \n",
       "2         2019-07-16T17:00:00.000Z  2019-07-16T17:00:00.000Z         2   \n",
       "3         2019-10-17T17:00:00.000Z  2019-10-17T17:00:00.000Z         2   \n",
       "4         2018-04-18T17:00:00.000Z  2018-04-18T17:00:00.000Z         2   \n",
       "...                            ...                       ...       ...   \n",
       "11401877  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1   \n",
       "11401878  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1   \n",
       "11401879  2020-12-30T17:00:00.000Z  2021-01-06T17:00:00.000Z         1   \n",
       "11401880  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1   \n",
       "11401881  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1   \n",
       "\n",
       "               biaya  jenispulang         cbg  kelasrawat  label  zs_biaya  \n",
       "0           184300.0          1.0    Q-5-42-0           3      0 -0.375566  \n",
       "1         10628400.0          1.0  D-4-13-III           1      0  1.634214  \n",
       "2           187300.0          1.0    Q-5-44-0           3      0 -0.374989  \n",
       "3           187300.0          1.0    Q-5-44-0           3      0 -0.374989  \n",
       "4           381600.0          1.0    Q-5-44-0           3      0 -0.337599  \n",
       "...              ...          ...         ...         ...    ...       ...  \n",
       "11401877   3645100.0          1.0    P-8-17-I           3      0  0.290403  \n",
       "11401878  12273200.0          1.0   P-8-13-II           2      0  1.950726  \n",
       "11401879   4099000.0          1.0    P-8-08-I           3      0  0.377748  \n",
       "11401880   3992400.0          1.0    P-8-08-I           3      0  0.357234  \n",
       "11401881   1624900.0          1.0    P-8-16-I           3      0 -0.098349  \n",
       "\n",
       "[11401590 rows x 16 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculating zscore in biaya column\n",
    "samp[\"zs_biaya\"] = (samp[\"biaya\"] - samp[\"biaya\"].mean())/samp[\"biaya\"].std()\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "109826\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>pisat</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "      <th>kelasrawat</th>\n",
       "      <th>label</th>\n",
       "      <th>zs_biaya</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-42-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.375566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-05-27T17:00:00.000Z</td>\n",
       "      <td>2019-05-30T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>D-4-13-III</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.634214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.374989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.374989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.337599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401877</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>301</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.290403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401878</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>233</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-13-II</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.950726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401879</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>118</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-06T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.377748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401880</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>101</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.357234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401881</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>12</td>\n",
       "      <td>SD</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-16-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.098349</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  dati2 typefaskes  usia jenkel  pisat  \\\n",
       "0           165666         486     17         KL    48      P    1.0   \n",
       "1          1010828         520     17          A    63      L    1.0   \n",
       "2           166042         523     17         KL    53      P    1.0   \n",
       "3           168937         549     17         KL    54      P    1.0   \n",
       "4          1005899         549     17          A    53      P    1.0   \n",
       "...            ...         ...    ...        ...   ...    ...    ...   \n",
       "11401877   9983563     8527704    301          B     0      P    4.0   \n",
       "11401878  11053870     8527709    233          A     0      P    4.0   \n",
       "11401879   7461049     8527751    118          B     0      L    4.0   \n",
       "11401880   1075162     8527853    101         SC     0      L    5.0   \n",
       "11401881    102794     8527876     12         SD     0      P    4.0   \n",
       "\n",
       "                         tgldatang                 tglpulang  jenispel  \\\n",
       "0         2018-07-25T17:00:00.000Z  2018-07-25T17:00:00.000Z         2   \n",
       "1         2019-05-27T17:00:00.000Z  2019-05-30T17:00:00.000Z         1   \n",
       "2         2019-07-16T17:00:00.000Z  2019-07-16T17:00:00.000Z         2   \n",
       "3         2019-10-17T17:00:00.000Z  2019-10-17T17:00:00.000Z         2   \n",
       "4         2018-04-18T17:00:00.000Z  2018-04-18T17:00:00.000Z         2   \n",
       "...                            ...                       ...       ...   \n",
       "11401877  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1   \n",
       "11401878  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1   \n",
       "11401879  2020-12-30T17:00:00.000Z  2021-01-06T17:00:00.000Z         1   \n",
       "11401880  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1   \n",
       "11401881  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1   \n",
       "\n",
       "               biaya  jenispulang         cbg  kelasrawat  label  zs_biaya  \n",
       "0           184300.0          1.0    Q-5-42-0           3      0 -0.375566  \n",
       "1         10628400.0          1.0  D-4-13-III           1      0  1.634214  \n",
       "2           187300.0          1.0    Q-5-44-0           3      0 -0.374989  \n",
       "3           187300.0          1.0    Q-5-44-0           3      0 -0.374989  \n",
       "4           381600.0          1.0    Q-5-44-0           3      0 -0.337599  \n",
       "...              ...          ...         ...         ...    ...       ...  \n",
       "11401877   3645100.0          1.0    P-8-17-I           3      0  0.290403  \n",
       "11401878  12273200.0          1.0   P-8-13-II           2      0  1.950726  \n",
       "11401879   4099000.0          1.0    P-8-08-I           3      0  0.377748  \n",
       "11401880   3992400.0          1.0    P-8-08-I           3      0  0.357234  \n",
       "11401881   1624900.0          1.0    P-8-16-I           3      0 -0.098349  \n",
       "\n",
       "[11291764 rows x 16 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# zscore is used to drop outlier in here im using -3 to 3 zscore data\n",
    "index_to_drop = []\n",
    "index_to_drop = list(samp[samp[\"zs_biaya\"] > 3].index)\n",
    "index_to_drop += list(samp[samp[\"zs_biaya\"] < -3].index)\n",
    "print(len(index_to_drop))\n",
    "samp.drop(index=index_to_drop, inplace=True)\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# removing zs_biaya column\n",
    "samp.drop(\"zs_biaya\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>pisat</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "      <th>kelasrawat</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-42-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-05-27T17:00:00.000Z</td>\n",
       "      <td>2019-05-30T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>D-4-13-III</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401877</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>301</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401878</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>233</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-13-II</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401879</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>118</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-06T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401880</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>101</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11401881</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>12</td>\n",
       "      <td>SD</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-16-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  dati2 typefaskes  usia jenkel  pisat  \\\n",
       "0           165666         486     17         KL    48      P    1.0   \n",
       "1          1010828         520     17          A    63      L    1.0   \n",
       "2           166042         523     17         KL    53      P    1.0   \n",
       "3           168937         549     17         KL    54      P    1.0   \n",
       "4          1005899         549     17          A    53      P    1.0   \n",
       "...            ...         ...    ...        ...   ...    ...    ...   \n",
       "11401877   9983563     8527704    301          B     0      P    4.0   \n",
       "11401878  11053870     8527709    233          A     0      P    4.0   \n",
       "11401879   7461049     8527751    118          B     0      L    4.0   \n",
       "11401880   1075162     8527853    101         SC     0      L    5.0   \n",
       "11401881    102794     8527876     12         SD     0      P    4.0   \n",
       "\n",
       "                         tgldatang                 tglpulang  jenispel  \\\n",
       "0         2018-07-25T17:00:00.000Z  2018-07-25T17:00:00.000Z         2   \n",
       "1         2019-05-27T17:00:00.000Z  2019-05-30T17:00:00.000Z         1   \n",
       "2         2019-07-16T17:00:00.000Z  2019-07-16T17:00:00.000Z         2   \n",
       "3         2019-10-17T17:00:00.000Z  2019-10-17T17:00:00.000Z         2   \n",
       "4         2018-04-18T17:00:00.000Z  2018-04-18T17:00:00.000Z         2   \n",
       "...                            ...                       ...       ...   \n",
       "11401877  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1   \n",
       "11401878  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1   \n",
       "11401879  2020-12-30T17:00:00.000Z  2021-01-06T17:00:00.000Z         1   \n",
       "11401880  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1   \n",
       "11401881  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1   \n",
       "\n",
       "               biaya  jenispulang         cbg  kelasrawat  label  \n",
       "0           184300.0          1.0    Q-5-42-0           3      0  \n",
       "1         10628400.0          1.0  D-4-13-III           1      0  \n",
       "2           187300.0          1.0    Q-5-44-0           3      0  \n",
       "3           187300.0          1.0    Q-5-44-0           3      0  \n",
       "4           381600.0          1.0    Q-5-44-0           3      0  \n",
       "...              ...          ...         ...         ...    ...  \n",
       "11401877   3645100.0          1.0    P-8-17-I           3      0  \n",
       "11401878  12273200.0          1.0   P-8-13-II           2      0  \n",
       "11401879   4099000.0          1.0    P-8-08-I           3      0  \n",
       "11401880   3992400.0          1.0    P-8-08-I           3      0  \n",
       "11401881   1624900.0          1.0    P-8-16-I           3      0  \n",
       "\n",
       "[11291764 rows x 15 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tidy up all string column to uppercase and remove whitespace\n",
    "samp[\"typefaskes\"] = samp[\"typefaskes\"].str.upper()\n",
    "samp[\"typefaskes\"] = samp[\"typefaskes\"].str.replace(\" \", \"\")\n",
    "samp[\"jenkel\"] = samp[\"jenkel\"].str.upper()\n",
    "samp[\"jenkel\"] = samp[\"jenkel\"].str.replace(\" \", \"\")\n",
    "samp[\"cbg\"] = samp[\"cbg\"].str.upper()\n",
    "samp[\"cbg\"] = samp[\"cbg\"].str.replace(\" \", \"\")\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save cleaned sampling to some folder\n",
    "samp.to_csv(\"temp_data/1/train_samp_cleaned.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>69.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202866</th>\n",
       "      <td>11426915</td>\n",
       "      <td>93.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202867</th>\n",
       "      <td>11426927</td>\n",
       "      <td>89.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202868</th>\n",
       "      <td>11426927</td>\n",
       "      <td>96.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202869</th>\n",
       "      <td>11426927</td>\n",
       "      <td>87.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202870</th>\n",
       "      <td>11426927</td>\n",
       "      <td>93.96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12202871 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id   proc\n",
       "0                6  90.59\n",
       "1                6  69.01\n",
       "2                6  88.78\n",
       "3               27  90.59\n",
       "4               27  88.78\n",
       "...            ...    ...\n",
       "12202866  11426915  93.57\n",
       "12202867  11426927  89.52\n",
       "12202868  11426927  96.55\n",
       "12202869  11426927  87.39\n",
       "12202870  11426927  93.96\n",
       "\n",
       "[12202871 rows x 2 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading procedure dataset\n",
    "import pandas as pd\n",
    "import gc\n",
    "proc = pd.read_csv(\"Dataset/sampling_healthkathon2022_procedure.csv\")\n",
    "proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id      0\n",
       "proc    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if theres any null value\n",
    "proc.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "301. - could not convert string to float: '-'\n",
      "\t82708\n",
      "2681. A59.8 could not convert string to float: 'A59.8'\n",
      "\t6\n",
      "3121. V58.2 could not convert string to float: 'V58.2'\n",
      "\t1\n",
      "3254. G54.1 could not convert string to float: 'G54.1'\n",
      "\t1\n",
      "3346. D72.9 could not convert string to float: 'D72.9'\n",
      "\t1\n",
      "3403. M17.1 could not convert string to float: 'M17.1'\n",
      "\t1\n",
      "3404. Z09.8 could not convert string to float: 'Z09.8'\n",
      "\t1\n",
      "3465. O80.9 could not convert string to float: 'O80.9'\n",
      "\t1\n",
      "3477. S90.3 could not convert string to float: 'S90.3'\n",
      "\t1\n",
      "3496. Z37.0 could not convert string to float: 'Z37.0'\n",
      "\t1\n",
      "3545. J44.9 could not convert string to float: 'J44.9'\n",
      "\t1\n",
      "3556. G93.4 could not convert string to float: 'G93.4'\n",
      "\t1\n",
      "3592. I21.4 could not convert string to float: 'I21.4'\n"
     ]
    }
   ],
   "source": [
    "# check if there is any inconsistent value\n",
    "# result there is inconsistent value\n",
    "for index, val in enumerate(proc[\"proc\"].unique().tolist()):\n",
    "    try:\n",
    "        float(val)\n",
    "    except Exception as e:\n",
    "        print(f\"{index}. {val} {e}\")\n",
    "        print(f\"\\t{len(proc[proc['proc'] == val])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2680. A59.8 could not convert string to float: 'A59.8'\n",
      "\t6\n",
      "3120. V58.2 could not convert string to float: 'V58.2'\n",
      "\t1\n",
      "3253. G54.1 could not convert string to float: 'G54.1'\n",
      "\t1\n",
      "3345. D72.9 could not convert string to float: 'D72.9'\n",
      "\t1\n",
      "3402. M17.1 could not convert string to float: 'M17.1'\n",
      "\t1\n",
      "3403. Z09.8 could not convert string to float: 'Z09.8'\n",
      "\t1\n",
      "3464. O80.9 could not convert string to float: 'O80.9'\n",
      "\t1\n",
      "3476. S90.3 could not convert string to float: 'S90.3'\n",
      "\t1\n",
      "3495. Z37.0 could not convert string to float: 'Z37.0'\n",
      "\t1\n",
      "3544. J44.9 could not convert string to float: 'J44.9'\n",
      "\t1\n",
      "3555. G93.4 could not convert string to float: 'G93.4'\n",
      "\t1\n",
      "3591. I21.4 could not convert string to float: 'I21.4'\n",
      "\t2\n",
      "3592. E11.9 could not convert string to float: 'E11.9'\n",
      "\t2\n",
      "3593. Q00.0 could not convert string to float: 'Q00.0'\n",
      "\t1\n",
      "3594. P59.9 could not convert string to float: 'P59.9'\n",
      "\t1\n",
      "3635. G62.9 could not convert string to float: 'G62.9'\n",
      "\t1\n",
      "3640. Z04.8 could not convert string to float: 'Z04.8'\n",
      "\t1\n",
      "3644. A24.0 could not convert string to float: 'A24.0'\n",
      "\t1\n",
      "3645. A31.1 could not convert string to float: 'A31.1'\n",
      "\t1\n",
      "3670. Z27.1 could not convert string to float: 'Z27.1'\n",
      "\t1\n",
      "3671. Z24.0 could not convert string to float: 'Z24.0'\n",
      "\t1\n",
      "3691. J81 could not convert string to float: 'J81'\n",
      "\t1\n"
     ]
    }
   ],
   "source": [
    "# dropping that has \"-\" value\n",
    "# keeping all other string because i think there is procedure but miss typed it\n",
    "# because im only using the amount of procedure its not a problem just count it as 1\n",
    "index_to_drop = list(proc[proc['proc'].str.contains('[-]', na=False)].index)\n",
    "proc.drop(index=index_to_drop, inplace=True)\n",
    "\n",
    "# check again is inconsitent value resolved\n",
    "for index, val in enumerate(proc[\"proc\"].unique().tolist()):\n",
    "    try:\n",
    "        float(val)\n",
    "    except Exception as e:\n",
    "        print(f\"{index}. {val} {e}\")\n",
    "        print(f\"\\t{len(proc[proc['proc'] == val])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>69.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202866</th>\n",
       "      <td>11426915</td>\n",
       "      <td>93.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202867</th>\n",
       "      <td>11426927</td>\n",
       "      <td>89.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202868</th>\n",
       "      <td>11426927</td>\n",
       "      <td>96.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202869</th>\n",
       "      <td>11426927</td>\n",
       "      <td>87.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12202870</th>\n",
       "      <td>11426927</td>\n",
       "      <td>93.96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12120163 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id   proc\n",
       "0                6  90.59\n",
       "1                6  69.01\n",
       "2                6  88.78\n",
       "3               27  90.59\n",
       "4               27  88.78\n",
       "...            ...    ...\n",
       "12202866  11426915  93.57\n",
       "12202867  11426927  89.52\n",
       "12202868  11426927  96.55\n",
       "12202869  11426927  87.39\n",
       "12202870  11426927  93.96\n",
       "\n",
       "[12120163 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save to some folder\n",
    "proc.to_csv(\"temp_data/1/train_proc_cleaned.csv\", index=False)\n",
    "proc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>diag</th>\n",
       "      <th>levelid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>O06.9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>57</td>\n",
       "      <td>J02.9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>91</td>\n",
       "      <td>R10.1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>109</td>\n",
       "      <td>R18</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>111</td>\n",
       "      <td>N81.3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17308531</th>\n",
       "      <td>11426947</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17308532</th>\n",
       "      <td>11426947</td>\n",
       "      <td>E78.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17308533</th>\n",
       "      <td>11426947</td>\n",
       "      <td>I21.9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17308534</th>\n",
       "      <td>11426952</td>\n",
       "      <td>K30</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17308535</th>\n",
       "      <td>11426952</td>\n",
       "      <td>M54.5</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17308536 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id   diag  levelid\n",
       "0                6  O06.9        1\n",
       "1               57  J02.9        1\n",
       "2               91  R10.1        1\n",
       "3              109    R18        1\n",
       "4              111  N81.3        1\n",
       "...            ...    ...      ...\n",
       "17308531  11426947    K30        2\n",
       "17308532  11426947  E78.0        2\n",
       "17308533  11426947  I21.9        1\n",
       "17308534  11426952    K30        1\n",
       "17308535  11426952  M54.5        2\n",
       "\n",
       "[17308536 rows x 3 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading diagnosa data\n",
    "import pandas as pd\n",
    "import gc\n",
    "diag = pd.read_csv(\"dataset/sampling_healthkathon2022_diagnosa.csv\")\n",
    "diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>diag</th>\n",
       "      <th>levelid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>111</td>\n",
       "      <td>O86.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>133</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>231</td>\n",
       "      <td>E11.9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>231</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>248</td>\n",
       "      <td>O82.1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17308528</th>\n",
       "      <td>11426932</td>\n",
       "      <td>M79.1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17308530</th>\n",
       "      <td>11426946</td>\n",
       "      <td>N30.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17308531</th>\n",
       "      <td>11426947</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17308532</th>\n",
       "      <td>11426947</td>\n",
       "      <td>E78.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17308535</th>\n",
       "      <td>11426952</td>\n",
       "      <td>M54.5</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5905255 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id   diag  levelid\n",
       "5              111  O86.0        2\n",
       "7              133    K30        2\n",
       "19             231  E11.9        2\n",
       "20             231    K30        2\n",
       "24             248  O82.1        2\n",
       "...            ...    ...      ...\n",
       "17308528  11426932  M79.1        2\n",
       "17308530  11426946  N30.0        2\n",
       "17308531  11426947    K30        2\n",
       "17308532  11426947  E78.0        2\n",
       "17308535  11426952  M54.5        2\n",
       "\n",
       "[5905255 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dropping all levelid 1 because im only use the amount of secondary diagnose\n",
    "# or the levelid 2 as a feature\n",
    "index_to_drop = []\n",
    "index_to_drop = diag[diag[\"levelid\"] == 1].index\n",
    "diag.drop(index=index_to_drop, inplace=True)\n",
    "diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uppercase al string in diag column and remove whitespace\n",
    "diag[\"diag\"] = diag[\"diag\"].str.upper()\n",
    "diag[\"diag\"] = diag[\"diag\"].str.replace(\" \", \"\")\n",
    "diag.to_csv(\"temp_data/1/train_diag_cleaned.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploratory Data Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>pisat</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "      <th>kelasrawat</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2018-07-25T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-42-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-05-27T17:00:00.000Z</td>\n",
       "      <td>2019-05-30T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>D-4-13-III</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2019-07-16T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>KL</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2019-10-17T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2018-04-18T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>301</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>233</td>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-03T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-13-II</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>118</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-06T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>101</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-08-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>12</td>\n",
       "      <td>SD</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P-8-16-I</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  dati2 typefaskes  usia jenkel  pisat  \\\n",
       "0           165666         486     17         KL    48      P    1.0   \n",
       "1          1010828         520     17          A    63      L    1.0   \n",
       "2           166042         523     17         KL    53      P    1.0   \n",
       "3           168937         549     17         KL    54      P    1.0   \n",
       "4          1005899         549     17          A    53      P    1.0   \n",
       "...            ...         ...    ...        ...   ...    ...    ...   \n",
       "11291759   9983563     8527704    301          B     0      P    4.0   \n",
       "11291760  11053870     8527709    233          A     0      P    4.0   \n",
       "11291761   7461049     8527751    118          B     0      L    4.0   \n",
       "11291762   1075162     8527853    101         SC     0      L    5.0   \n",
       "11291763    102794     8527876     12         SD     0      P    4.0   \n",
       "\n",
       "                         tgldatang                 tglpulang  jenispel  \\\n",
       "0         2018-07-25T17:00:00.000Z  2018-07-25T17:00:00.000Z         2   \n",
       "1         2019-05-27T17:00:00.000Z  2019-05-30T17:00:00.000Z         1   \n",
       "2         2019-07-16T17:00:00.000Z  2019-07-16T17:00:00.000Z         2   \n",
       "3         2019-10-17T17:00:00.000Z  2019-10-17T17:00:00.000Z         2   \n",
       "4         2018-04-18T17:00:00.000Z  2018-04-18T17:00:00.000Z         2   \n",
       "...                            ...                       ...       ...   \n",
       "11291759  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1   \n",
       "11291760  2020-12-30T17:00:00.000Z  2021-01-03T17:00:00.000Z         1   \n",
       "11291761  2020-12-30T17:00:00.000Z  2021-01-06T17:00:00.000Z         1   \n",
       "11291762  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1   \n",
       "11291763  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1   \n",
       "\n",
       "               biaya  jenispulang         cbg  kelasrawat  label  \n",
       "0           184300.0          1.0    Q-5-42-0           3      0  \n",
       "1         10628400.0          1.0  D-4-13-III           1      0  \n",
       "2           187300.0          1.0    Q-5-44-0           3      0  \n",
       "3           187300.0          1.0    Q-5-44-0           3      0  \n",
       "4           381600.0          1.0    Q-5-44-0           3      0  \n",
       "...              ...          ...         ...         ...    ...  \n",
       "11291759   3645100.0          1.0    P-8-17-I           3      0  \n",
       "11291760  12273200.0          1.0   P-8-13-II           2      0  \n",
       "11291761   4099000.0          1.0    P-8-08-I           3      0  \n",
       "11291762   3992400.0          1.0    P-8-08-I           3      0  \n",
       "11291763   1624900.0          1.0    P-8-16-I           3      0  \n",
       "\n",
       "[11291764 rows x 15 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading samp data\n",
    "import pandas as pd\n",
    "\n",
    "samp = pd.read_csv(\"temp_data/1/train_samp_cleaned.csv\")\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "id_peserta     0\n",
       "dati2          0\n",
       "typefaskes     0\n",
       "usia           0\n",
       "jenkel         0\n",
       "pisat          0\n",
       "tgldatang      0\n",
       "tglpulang      0\n",
       "jenispel       0\n",
       "biaya          0\n",
       "jenispulang    0\n",
       "cbg            0\n",
       "kelasrawat     0\n",
       "label          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the null\n",
    "samp.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>usia</th>\n",
       "      <th>pisat</th>\n",
       "      <th>nday</th>\n",
       "      <th>kelasrawat</th>\n",
       "      <th>biaya</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>48</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>63</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>53</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>54</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>53</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          usia  pisat  nday  kelasrawat       biaya  label\n",
       "0           48    1.0     0           3    184300.0      0\n",
       "1           63    1.0     3           1  10628400.0      0\n",
       "2           53    1.0     0           3    187300.0      0\n",
       "3           54    1.0     0           3    187300.0      0\n",
       "4           53    1.0     0           3    381600.0      0\n",
       "...        ...    ...   ...         ...         ...    ...\n",
       "11291759     0    4.0     4           3   3645100.0      0\n",
       "11291760     0    4.0     4           2  12273200.0      0\n",
       "11291761     0    4.0     7           3   4099000.0      0\n",
       "11291762     0    5.0     2           3   3992400.0      0\n",
       "11291763     0    4.0     2           3   1624900.0      0\n",
       "\n",
       "[11291764 rows x 6 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creating nday column based on tglpulang-tgldatang in the result is int of day\n",
    "# save all numerical data to a variable for checking correlation\n",
    "\n",
    "samp[\"tgldatang\"] = pd.to_datetime(samp[\"tgldatang\"], format=\"%Y/%m/%dT%H:%M:%S\")\n",
    "samp[\"tglpulang\"] = pd.to_datetime(samp[\"tglpulang\"], format=\"%Y/%m/%dT%H:%M:%S\")\n",
    "samp[\"nday\"] = samp[\"tglpulang\"] - samp[\"tgldatang\"]\n",
    "samp[\"nday\"] = pd.to_timedelta(samp[\"nday\"]).dt.days\n",
    "numeric_col = samp[[\"usia\", \"pisat\",\"nday\", \"kelasrawat\", \"biaya\", \"label\"]]\n",
    "numeric_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot: >"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGiCAYAAAB6c8WBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAACLM0lEQVR4nOzdd1wURxvA8d+hUgVFaaKIvUTFgoq91yjGEltiQ41J7GI39hKNvcSusXcTexfFXhAEC4oVUWkiCooKCvf+wevpURSOo8nzzWc/kdnZuWeW3WVudnZWoVQqlQghhBAiy9JJ7wCEEEIIkb6kMSCEEEJkcdIYEEIIIbI4aQwIIYQQWZw0BoQQQogsThoDQgghRBYnjQEhhBAii5PGgBBCCJHFSWNACCGEyOKkMSCEEEJkcdIYEEIIITKI06dP4+joiLW1NQqFgt27d391G1dXVypVqoSenh7FihVj7dq1yf5caQwIIYQQGURERATly5dn8eLFScr/8OFDWrRoQf369fH09GTw4MH07t2bI0eOJOtzFfKiIiGEECLjUSgU7Nq1i9atWyeaZ+TIkRw4cIAbN26o0jp16sTLly85fPhwkj9LegaEEEKIVBQZGUl4eLjaEhkZqZWyL1y4QKNGjdTSmjZtyoULF5JVTnatRKMF70MepHcIGYKBde30DkFkINl0pL0OEB0Tk94hiAzmQ9TTVC1fm3+Tpv+9nkmTJqmlTZgwgYkTJ6a47MDAQCwtLdXSLC0tCQ8P5+3btxgYGCSpnAzTGBBCCCEyjJhorRU1evRonJ2d1dL09PS0Vr42SGNACCGESEV6enqp9sffysqKoKAgtbSgoCBMTEyS3CsA0hgQQggh4lNmjltT1atX5+DBg2ppx44do3r16skqR25ICiGEEHHFxGhvSYbXr1/j6emJp6cnEPvooKenJ35+fkDsLYdu3bqp8v/22288ePCAESNGcPv2bZYsWcL27dsZMmRIsj5XegaEEEKIOJTp1DNw5coV6tevr/r541iD7t27s3btWgICAlQNA4DChQtz4MABhgwZwoIFCyhQoACrVq2iadOmyfrcDDPPgDxNEEueJhCfk6cJYsnTBCKu1H6aIMr/ptbK0rUuo7WyUov0DAghhBBxZbEGqDQGhBBCiLgyyQBCbZE+SCGEECKLk54BIYQQIi4tTjqUGUhjQAghhIhLbhMIIYQQIiuRngEhhBAiLnmaQAghhMja0mvSofQitwmEEEKILE56BoQQQoi45DaBEEIIkcVlsdsE0hgQQggh4spi8wzImAEhhBAii0txz8C7d++IiopSSzMxMUlpsUIIIUT6kdsEX/fmzRtGjBjB9u3bef78ebz10dFZq3tFCCHENyaLDSDU6DbB8OHDOXHiBEuXLkVPT49Vq1YxadIkrK2tWb9+vbZjFEIIIUQq0qhnYN++faxfv5569erh5ORE7dq1KVasGLa2tmzatImff/5Z23EKIYQQaSeL3SbQqGcgNDSUIkWKALHjA0JDQwGoVasWp0+f1l50QgghRHqIidHekglo1BgoUqQIDx8+BKBUqVJs374diO0xyJ07t9aCSwtXPK/Tb8QE6rf6mbI1m+Ny+nx6h5QqJk4YxuNHHrwKu8eRQ1spVqzwF/OPH+fMh6inasuN66dU621tC8Rb/3Fp165lalcnSZJbZ4Dff+vOvTsXeR1+n/Nn91GlcgXVOlPT3MyfN4WbN07zKuweD+5dZt7cyZiYGKuVUdm+PEcPbyMk2JtnQTc5uH8Tdnbfabt6SfLbr93x8TlP2Mu7nDm9l8qf1Schbdu24JrXScJe3sX9yjGaNa2vtv6HH5pxYP8m/J9eI/Ld4wTrdfTodiLfPVZb/l70pzarpTFtHxPJOQ+6de2Ah/sxXoffx/+JFwsXTNN29ZJM2/sBoHevn3E5toPQkNt8iHpKrlzxB5KPHjWQM6f2EP7yHiHB3tqqjtACjRoDTk5OeHl5ATBq1CgWL16Mvr4+Q4YMYfjw4VoNMLW9ffuOksWK8MfQvukdSqoZPqwv/fv1pG//UdSo5UjEmzcc3L8JPT29L2534+Zt8ttUUC1167VWrXv82F9tXX6bCkycNItXr15z+PCJVK7R12lS5/btWzF71gSmTJ1LFYdmeF3z5uCBTZib5wXA2toSa2tLRo6cQvmKDenVewhNm9Zn5Yo5qjKMjAw5sH8Tfo+fUqOWI3Xrt+HV6wgO7t9E9uxpO63Hjz86MnPmOKZNm49Dte+5ft2b/fs2qOoTV7Vq9mxY/zdr127FwaE5e/cdYceOVXz3XUlVHiMjQ86dv8wfY7/8x3316k0UtK2kWkaPSf/GQGocE0k9DwYP6sOUySOYOWsxdhUa0LR5J44ec03tKicoNfYDgKGhAUeOujLjr0WJlqOrm4Od/+5n+fKMP7ZMqYzW2pIZKJRKpTKlhTx69Ah3d3eKFSuGnZ2dRmW8D3mQ0jBSrGzN5iyYPo6GdWqkWwwG1rW1XubjRx7Mm7+cufOWA2BiYoz/E0969h7C9u17E9xm/DhnWrVqRuUqTZL8OW6Xj3D16nX6/DpMK3GnhCZ1Pn92H25XvBg0eCwACoUC3wduLF6yhpmzFie4Tbt2LVm/diEmuYsTHR2NfSU7Ll08RKEiVXjyxB+AsmVL4enhQsnSNbl/3zdZ9cimo/lUIGdO78Xd3YvBQ8ap6nP/3mWWLF3D7NlL4uXfuGEJRkYGtGnrpEo7fWoP167dpP+AMWp5bW0LcMfnAlWqNuXaNfVveEePbuea102GDZ+kcexxRWuhqzWtjom450Hu3Lnw83WndZsenDh5NsX1SKnU3g9161TH5fhO8pqXJiwsPMHyunXtwNw5EzGz0LzH7EPUU423TYp3nvu1VpZ+hYzRW/olWpl0yNbWlrZt22rcEBCpp3DhguTLZ4nLiU8XofDwV1y+fJVqDvZf3LZ4scL4+bpz5/Z51q9bhI2NdaJ5K1UsR8UKZVmzZqvWYteUJnXOkSMHlSrZ4XLijCpNqVTicuIs1aolvp9ymRgTHv5a9Titz537hISE0tOpEzly5EBfXx+nHp3xvnUHX9/HWqrh18XWpxwnPtsHSqWSEyfPJLoPHKpVUssPcOz4KRy+cpwkpFOnNjx94oWH+3GmTBmJgYF+ssvQprQ6JhI6Dxo1qoOOjgLr/FZcv+aK74MrbNm8jAIFEj+fUktanhsic0lyv+XChQvp06cP+vr6LFy48It5Bw4c+MX1kZGRREZGqqXpREZ+tdtaJJ+VpQUAQUHP1NKDgkOwsrJIdLvLl6/Ss/cQ7ty5Tz4rC8aNdcb1xC7KV2zA69cR8fI7OcX+wbtw8Yp2K6ABTepsZpaH7NmzExwUopYeHPyMUiWLJrhN3rym/DFmMKtWb1KlvX4dQcPGP/LvjtX8MWYwAHfvPeT7Fj+l6fwbH+sTFKy+D4KDQihZoliC21hZmhMUHBIvv6WlebI+e9u23fg9eop/QBDlypVi2tQxlChelI6d+iSvElqUVsdEQudBkcIF0dHRYdTIATg7jycs7BWTJ43g8KEtVKzUiPfv36ekasmSVvvhm5BJBv5pS5IbA/PmzePnn39GX1+fefPmJZpPoVB8tTEwffp0Jk1S70IcO3wg40cMSmo4IhGdO7dh6eK/VD+3+qGbRuUcPnJS9e/r129x6fJVHty7RPsfHVmzVv3bv76+Pp07tWbanws0CzqFtFXn5DA2zsm+Peu5desOkyZ/GjOgr6/PyuWzOX/hCl269iNbtmw4O//G3j3rqVa9Be/evUv12NLb6tWbVf++efM2gYHBHDm8jSJFbHnw4FGaxJAex0Ri54GOjg66uroMGTKOY8djn7b6uWtfnj72pH69Ghw9diqh4rQiPfbDNyOLPVqY5MbAx6cH4v5bE6NHj8bZ2VktTedV6t7/ySr27TvK5ctXVT/r6ekCYGlpTmBgsCrd0sIMT6+bSS43LCycO3cfUKxYoXjr2rVrgaGhARs27tA88BTQRp1DQkL58OEDFpZmaukWFuYExvkWlTOnEQf3b+LVqwjate/Nhw8fVOs6d2qNra0NNWu34uNwnC5d+xES7E2rVk0SvSerbR/rY2mh/q3ewtIs3rfCjwKDnmFpYZbk/En18XdTtEihNGsMpPUxAYmfBwEBQQB437qrVnZISCg2NvmTWbPkSY/98M2QFxUlX3R0NJ6enrx48SJJ+fX09DAxMVFb5BaBdrx+HcH9+76qxdv7DgEBQTSoX0uVx9g4J1WrVuTiJfckl2tkZEjRIrYEBATHW9ezRyf27T9GSEioVuqQXNqo8/v37/HwuKa2jUKhoEH9Wly8+GkbY+OcHD64haioKFq37RHvdpehoQExMTF8Pi734886KRgMmFyx9blO/fo1VWkKhYL69Wolug8uXfRQyw/QsEFtLiXjOElI+fJlAAgIDEpROcmRlsfER4mdB+cvxN4yKFniU5e6qWluzMzy4Of3JEX1/Jr02A8ic9Lo6jR48GBWr14NxDYE6tSpQ6VKlbCxscHV1VWb8aW6N2/ecvvOfW7fuQ/AU/8gbt+5T0Bg/D96mdXCRasYM3ogLVs2pmzZUqxdswB//yD27DmiynP08Db6/t5D9fPMGeOoU7satrYFqF6tMv/uWE10dAxbt+1WK7to0ULUrl2Nf/7ZTEaiSZ3nLVhJ714/0bVre0qVKsbiv2dgZGTA2nXbgE8NAUMjA375dRgmJsZYWppjaWmu+kN/3OU0pqa5WLTwT0qVKsZ335Vg9ap5fPjwAVfXtJ3DYsHClfTs2ZkuXX6kVMli/L3oT4yMDFi/PnZekNWr5zFlykhV/r8Xr6ZJk3oMHtSHkiWKMnbsEOzt7ViydJ0qj6lpbuzsvqN0qeIAlChRFDu771TjCooUsWX06EFUrFgOW9sCtGzRmH9Wz+f0mYvcuHE7DWsfX2ocEx996Ty4e/cBe/YeZu7cSVSvVpkyZUqy5p/53Pa5x8k0PiYg9faDpaU55cuXoWjRQgCUK1uK8uXLYGqaW5XHxsaa8uXLULCgNdmyZaN8+TKUL18GIyPD1K528iljtLdkAho9+Lxz5066dOkCxE405Ovry+3bt9mwYQN//PEH586d02qQqenG7bv0HPDpgjhz0QoAfmjeiGljh6ZXWFo1a/YSjIwMWbZkJrlzm3DunBstHLuofastUsQWM7M8qp/zF8jHxg2LyZvXlGfPQjl3/jI1azvG+9bj1KMTT54EpOp9T01oUucdO/ZibpaHieOHYWVljpfXTVq07ELw/wfVVapYDgeHSgDcua1+ES9a3IFHj57g43Of1m16MG6sM2dP7yUmJgZPz9hyAtO4gblz5z7MzfIwfvxQrCzN8fLyxrFVV1V9bGzyExPzqQfj4kV3unUfwKSJw5k8eQT37vnSvn1vvL19VHlatmzMqpVzVT9v2hj7iOKUqXOZOnUeUVFRNGhQiwH9e2FkZMCTJwHs2nWQ6TO+POg4LaTGMfHR186DHk6DmDN7Inv3rCMmRsnpMxdo0bKL2i2mtJJa++HXPl0ZP+7TNdP15C4AevYawvoNsQ3QiROG071bB1Ued7ejADRs9COnTl9InQprKosNINRongF9fX3u3btHgQIF6NOnD4aGhsyfP5+HDx9Svnx5wsMTfrb0SzLCPAMZQWrMMyAyr5TMM/At0cY8A+LbkurzDFzc9vVMSaRfraPWykotGl1pLC0t8fb2Jjo6msOHD9O4cWMg9tXG2bJl02qAQgghRJqT2wRf5+TkRIcOHciXLx8KhYJGjRoBcOnSJUqVKqXVAIUQQog0l8V6ozRqDEycOJFy5crh5+dH+/btVU8CZMuWjdGjR2s1QCGEEEKkLo0aA5MnT1b9+59//lFb9+jRI1q1apWyqIQQQoj0JD0DX7dr1y61n9+/f8/Dhw/Jnj07RYsWZfz48VoJTgghhEgPmeVtg9qiUWPg6tWr8dLCw8Pp0aMHbdq0SXFQQgghhEg7WntuycTEhEmTJjFu3DhtFSmEEEKkj5gY7S2ZgEY9A4kJCwsjLCxMm0UKIYQQaS+TPBKoLRo1BuK+wlipVBIQEMCGDRto3ry5VgITQggh0k0m+UavLRo1BuK+wlhHRwdzc3O6d+8ujxYKIYQQmYxGjYGUvsJYCCGEyNDkNoEQQgiRxWWx2wTyFhQhhBAii5OeASGEECIuuU0ghBBCZHFym0AIIYQQWYn0DAghhBBxZbGeAWkMCCGEEHFlsTEDcptACCGEyOKkZ0AIIYSIS24TCCGEEFlcFrtNII0BIYQQIq4s1jMgYwaEEEKILE56BoQQQoi45DaBEEIIkcVlsdsEGaYxYGBdO71DyBDe+p9J7xAyBLNCjdM7hAzh3Yeo9A4hQ8hrYJzeIWQIL969Tu8QxDcqwzQGhBBCiAxDegaEEEKILE6pTO8I0pQ8TSCEEEJkcdIzIIQQQsQltwmEEEKILC6LNQbkNoEQQgiRxUnPgBBCCBGXTDokhBBCZHFym0AIIYTI4pRK7S3JtHjxYgoVKoS+vj4ODg5cvnz5i/nnz59PyZIlMTAwwMbGhiFDhvDu3btkfaY0BoQQQogMYtu2bTg7OzNhwgQ8PDwoX748TZs2JTg4OMH8mzdvZtSoUUyYMIFbt26xevVqtm3bxpgxY5L1udIYEEIIIeKKidHekgxz587ll19+wcnJie+++45ly5ZhaGjIP//8k2D+8+fPU7NmTX766ScKFSpEkyZN6Ny581d7E+KSxoAQQggRlxYbA5GRkYSHh6stkZGR8T4yKioKd3d3GjVqpErT0dGhUaNGXLhwIcEwa9Sogbu7u+qP/4MHDzh48CDff/99sqorjQEhhBAiFU2fPp1cuXKpLdOnT4+XLyQkhOjoaCwtLdXSLS0tCQwMTLDsn376icmTJ1OrVi1y5MhB0aJFqVevntwmEEIIIVJMGaO1ZfTo0YSFhakto0eP1kqYrq6u/PnnnyxZsgQPDw/+++8/Dhw4wJQpU5JVjjxaKIQQQsShjNHei4r09PTQ09P7aj4zMzOyZctGUFCQWnpQUBBWVlYJbjNu3Di6du1K7969AShXrhwRERH06dOHP/74Ax2dpH3nl54BIYQQIgPQ1dXF3t4eFxcXVVpMTAwuLi5Ur149wW3evHkT7w9+tmzZAFAm47FG6RkQQggh4kqnSYecnZ3p3r07lStXpmrVqsyfP5+IiAicnJwA6NatG/nz51eNOXB0dGTu3LlUrFgRBwcH7t27x7hx43B0dFQ1CpJCo8ZAz549WbBgAcbGxmrpERERDBgwINFHIIQQQohMIZ2mI+7YsSPPnj1j/PjxBAYGUqFCBQ4fPqwaVOjn56fWEzB27FgUCgVjx47l6dOnmJub4+joyLRp05L1uQplcvoR/i9btmwEBARgYWGhlh4SEoKVlRUfPnxIbpFk182f7G2+RW/9z6R3CBmCWaHG6R1ChvDuQ1R6h5Ah5NYzSu8QMoQX716ndwgZRlTkk1Qt/83SAVory/D3RVorK7Ukq2cgPDwcpVKJUqnk1atX6Ovrq9ZFR0dz8ODBeA0EIYQQItPR4gDCzCBZjYHcuXOjUChQKBSUKFEi3nqFQsGkSZO0FpwQQgiRLrLYi4qS1Rg4efIkSqWSBg0a8O+//5InTx7VOl1dXWxtbbG2ttZ6kEIIIUSaksZA4urWrQvAw4cPsbGxSfLzi0IIIYTIuDR6msDW1haIfb7Rz8+PqCj1QU52dnYpj0wIIYRILxq8ejgz06gx8OzZM5ycnDh06FCC66Ojo1MUlBBCCJGusthtAo36+QcPHszLly+5dOkSBgYGHD58mHXr1lG8eHH27t2r7RhTZOKEYTx+5MGrsHscObSVYsUKfzH/+HHOfIh6qrbcuH5Ktd7WtkC89R+Xdu1apnZ1Us0Vz+v0GzGB+q1+pmzN5ricPp/eIaVI7z5duHbzFEEh3ric/JdK9l/urWrdpjluHkcJCvHm/KWDNG5ST239qDEDcfM4in/QdR499mDPvvXYVy6vWl+rtgNhr+8nuFSqVC41qpgkv/3aHR+f84S9vMuZ03upXLnCF/O3bduCa14nCXt5F/crx2jWtL7a+h9+aMaB/Zvwf3qNyHePsbP77ovl7d2znsh3j2nl2DSlVUkRp94/4XbNhUdBXhxy2UbFr/xOHFs35azbQR4FeeF6fi8NG9dRW79gyXSCwm6rLVv+XRmvnEZN6nLIZRu+gZ74PLrE2k1/a7VeX/Pbb92543OB8LB7nD2z76u//3ZtW3D9mivhYffwcD9Os2YN4uWZMH4Yj3zdCXt5j0OHtqhdU+vUqU5U5JMEF3v7T+fLj+1a4nb5CC9f3OXunYs4O/+mtToLzWjUGDhx4gRz586lcuXK6OjoYGtrS5cuXZg5c2aCb2JKL8OH9aV/v5707T+KGrUciXjzhoP7N311jugbN2+T36aCaqlbr7Vq3ePH/mrr8ttUYOKkWbx69ZrDh0+kco1Sz9u37yhZrAh/DO2b3qGkWNt2Lfhz+hj+mr6QOrVacePGbXbtXouZed4E81d1qMTqNfPZsG4HtWs6cmD/MTZvXUrp7z49MXPv7kOGO0+khsP3NG3SEb9HT9i1Zx15zWIH0V666EHxIg5qy7o12/B96IeHx/W0qHY8P/7oyMyZ45g2bT4O1b7n+nVv9u/bgHki+6FaNXs2rP+btWu34uDQnL37jrBjxyq++66kKo+RkSHnzl/mj7F/fvXzBw7onazpUFPLD22bM+nPUcz5azGN67Tl5g0ftu5ahZlZngTzV65akWWr57B5w04a1W7DoQPHWbv5b0qVLq6Wz+XYacoWr6Vafus1VG19i1ZN+HvFX2zZ9B8NarbGsclP/Ldzf6rVM672Pzoya+Z4pk6bh4NDc65d9+bA/o1f/v1vWMyatVup6tCMvXsPs3PHKsp89vsfNrQv/fo50X/AaGrVcuRNxBv279+ouqZeuHAFm4IV1ZbVqzfz4OEj3N29AGjatD7r1i1ixcoNVKzUkIGDxjBwQG9+/71Hqu+TZIlRam/JBDSadMjExIRr165RqFAhbG1t2bx5MzVr1uThw4eUKVOGN2/eJDuQ1Jh06PEjD+bNX87cecsBMDExxv+JJz17D2H79oR7MMaPc6ZVq2ZUrtIkyZ/jdvkIV69ep8+vw1Icc0aYdKhszeYsmD6OhnVqpFsMKZl0yOXkv3h4XGP40NjHXBUKBd4+Z1mxbD3z5i6Pl3/NuoUYGhrQsf0vqrTjJ3Zy/fothgwal+BnGBvn5EmAF61aduWUa/xelOzZs3P77nmWL1vPrL80/zaYkkmHzpzei7u7F4OHxNZBoVBw/95llixdw+zZS+Ll37hhCUZGBrRp66RKO31qD9eu3aT/APXXodraFuCOzwWqVG3KtWve8cqys/uOXf+tpUbNFvg98qB9+97s3XdE47qkZNKhQy7buOpxgzHDY9/iplAouOrtyuoVG1k0L/63+RVr5mJoaEiXjp++rR48vpUb128zYshEILZnIFcuY3r83D/Bz8yWLRtXrrswa/oiNm/4V+PY40rOpENnz+zjirsXgwePBWLr/eC+G0uWrGHW7MXx8m/auARDI0PatOmhSjtzei9e127Sv3/sG/Ye+bozf8EK5n12TX3y+Cq9ezuzfUf8a2r27NnxfXiFJUvW8Of0BQCsX/c3OXJkp/NPn/Zv375ODHX+naLFqia5fqk+6dCsnlory3B4xp+VV6OegZIlS+Lj4wNA+fLlWb58OU+fPmXZsmXky5dPqwFqqnDhguTLZ4nLibOqtPDwV1y+fJVqDvZf3LZ4scL4+bpz5/Z51q9bhI1N4o9LVqpYjooVyrJmzVatxS40lyNHDipULIvryU9/oJVKJa4nz1OlasUEt6lStSKuJ8+ppbm4nEk0f44cOejh1ImXL8O5fv1Wgnm+b9GQPHlys2nDTg1rkjI5cuSgUqVynPjs+FcqlZw4eSbR49+hWiW1/ADHjp/C4SvnS1wGBvqsX7eIwYPHEhT0LPnBa1GOHDmwq1CGM67qx8Np1wtUrlIhwW3sq1TgdJwG3kmXc/Hy16hVlZv3znHuyiH+mjsBU9PcqnV25b/DOr8VMTFKjp/5j2s+p9m8c0W83oXU8un3/+nLhVKp5MSJM1SrVinBbRwc7NXyAxw7dkp1vHy8pp5w+ZQn9prqiUO1hI8Rx5ZNyJvXlHXrt6vS9PR0efcuUi3f27fvsLGxxta2QPIqKrRGo8bAoEGDCAgIAGDChAkcOnSIggULsnDhQv788+vdh5GRkYSHh6st2u5OtLKMnQkx7sUoKDgEK6vEZ0m8fPkqPXsPoYVjF/oPGE3hQgVxPbGLnDkT/mbi5NQZ71t3uHDxivaCFxrLm9eU7NmzExwcopb+LDgES0vzBLextDQj+Nnzr+Zv2qw+TwOvEfzcm779nWjTqhuhz18kWGbXbh1wOX4Gf//AFNRGc2ZmeciePTtBwerHf3BQ4vvBytKcoDj77Uv5EzN71gQuXHRn3/6jyQs6FeT5//HwLDjO7/dZCBaWZgluY2Fp9tX8J13OMOC3kfzYyompE2ZTvWYVtvy7QvW4tW1hGwCGjerHvFnL6NLxd8JehvPfgfXkNs2lzSomSPX7j3P9Cw4OwdIy4euflZU5wUHqv/+g4Geq3//H/8c7RoKfYZXIMdLDqRNHj53i6dMAVdrRY6do3bo59evXRKFQULx4YYYM7vP/GDLQDLZZ7DaBRk8TdOnSRfVve3t7Hj16xO3btylYsCBmZgmfYJ+bPn16vJkKFTo5UWQz0SQcADp3bsPSxX+pfm71QzeNyjl85KTq39ev3+LS5as8uHeJ9j86smat+rd/fX19OndqzbQ/F2gWtMhUzpy+SO0ajuTJa0qPHh1Zu34RDeq3IyROQ8La2oqGjWrTo5v25jbPLFq2aEy9ejWp6tAsvUNJVbv/Paj69y3vO3jf9OGy13Fq1q7KmVMX0VHENgoWzFnOgb2xjaJBfUdz9dYpHFs3Y8OabekSd1rKnz8fTRrX5aeffldLX716E0WL2LJ71zpy5MhOePhr/v57NePHDyUmA/3hVMrTBMmnp6eHjo5Okl+XOHr0aMLCwtQWhY7x1zf8gn37jmJfpYlqCXkeChDvW42lhRmBgcFJLjcsLJw7dx9QrFiheOvatWuBoaEBGzbuSFHsQnueP3/Bhw8fsLBQb5SaW5gl2mUdFBSCRZxBVQnlf/PmLQ8ePOKKmyf9+43mw4dounVrH6+8n7v+SGjoSw4ecIm3Lq2EhITy4cMHLC3Uj38Ly8T3Q2DQMyzj7Lcv5U9IvXo1KFLEluCgm0S8fkjE64cAbN26nKNHt39la+0L/f/xYG4R5/drbhbvW/BHwUEhycoP8Mj3CSEhoRQqEjsHy8d95nP7nipPVNR7/HwfU6BA6t9KVf3+41z/LCzMCApK+PoXGPgsXm+JpYW5qi4f/x/vGLEwJzCBY6R7tw48f/4iwR6iMX/8iWmeEhQrXg2bghVxu+IJwMOHj5JWQaF1Gj9auHr1aiB2ToE6depQqVIlbGxscHV1/er2enp6mJiYqC0KhUKTUFRev47g/n1f1eLtfYeAgCAa1K+lymNsnJOqVSty8ZJ7kss1MjKkaBFbAgLin0A9e3Ri3/5jhISEpih2oT3v37/H8+oN6tb7NPhRoVBQt1513C5fTXAbt8tX1fID1K9fK9H8H+noKNDV042X3qVLO7Zu3qXR2zu15f3793h4XKd+/ZqqNIVCQf16tRI9/i9d9FDLD9CwQW0uJeN8mTV7CfaVm1ClajPVAjB8+CT69Bn6la217/3791zzvEntutVVaQqFgtp1q3HFzTPBbdzdPNXyA9StXyPR/AD5rC3Jkyc3wf//ouHleYN37yIpVvzTY3fZs2fHpmB+njz217xCSfTp9//p+qdQKKhfvxYXL3okuM2lS+5q10uAhg1rq46Xhw/9CAgIon6DuNfUCly6GP8Y6da9Axs37Uz0PIiJicHfP5D379/TscMPXLhwJWNdS+U2wdft3LlTdatg3759+Pr6cvv2bTZs2MAff/zBuXPnvlJC2li4aBVjRg/k7r0H+Po+ZtLE4fj7B7Fnz6dRzUcPb2P3nkMsWboWgJkzxrH/wDEe+T3BOp8VE8YPJTo6hq3bdquVXbRoIWrXroZjq65pWKPU8+bNW/yefLpIPfUP4vad++QyMSZfRrqPlwSL//6HpctncdXjOu7uXvTt54SRoSEbN8YO5lu2YjYB/oFMmjgbgKVL1nLw8Gb6D+jFkSMnafdjSypWKsuggX8AYGhowLDhfTl40IWgwGDy5s1D7z5dyGdtxe5d6hNv1a1Xg0KFC7JuXfp3Ay9YuJLVq+bi7nGNK26eDBjQCyMjA9b/fzDX6tXz8PcPZNy42Ntrfy9ezfFjOxg8qA+HDrnQvkMr7O3t6NtvlKpMU9Pc2NhYY50v9t3qJUoUBWK/NX6+xPX4sT++vo9Tu8oJWrZ4LQuXzsDz6g2uul+jT9/uGBoZsHXjfwAsWjaDwIBgpk2aC8CKpRvYfXA9v/V34vgRV1q3a0H5imUYNmg8AIZGhgwb1Y8De44SHBxCocI2jJs8nIcP/DjpEjsA8/WrCNb/s5Xhowfw9GkgT/z86TcodnT63t2H06TeCxasYPXqeXi4e+F2xZMBA3pjZGTAuvWxx+Y/q+fj7x/I2HEzYvfD36txOb6TwYNjf/8d2v8Q+/vvO1JV5qJFqxk9aiD37j3E9+FjJk4chn9AEHv2qj8pUr9+TYoUtmXNmi3x4sqb15S2bVtw+vQF9PX06Na9I+3ataRhox9TcW9oQJm1bhNo1BgICQnBysoKgIMHD9K+fXtKlChBz549WbAg49w/nzV7CUZGhixbMpPcuU04d86NFo5diIz8NJK1SBFbteeN8xfIx8YNi8mb15Rnz0I5d/4yNWs7xmuxOvXoxJMnARw9dopvwY3bd+k54NNJP3PRCgB+aN6IaWPT/htdSvz37wHymuVhzNjBWFqacf3aLdq2cVINCitgk4+Yz+4HXr7kQe+eQxg7zpnxE4dy//4jfur0O7e87wCxvV8lShal889tyZvXlNDQl3i4X6N5k47cvnVX7bO7dmvPxQvu3L3zIO0qnIidO/dhbpaH8eOHYmVpjpeXN46tuqoGV9rY5Fe7R3vxojvdug9g0sThTJ48gnv3fGnfvjfe3j6qPC1bNmbVyrmqnzdtjH1EccrUuUydOi+NapY8e/47RN68eRgxZgAWlubcvH6Lzm1/4dn/x3rkL2Ctth+uXL7K772HMWrsYMaMH8LD+770+Km/6ncdEx3Nd2VK0rFza0xyGRMY8IxTJ8/x19QFREW9V5UzadwsPkRHs3j5X+jr6+Ph7kU7xx6EvQxPk3rv2LkPM/O8jB8/DCur2N9/S8e4v/9P58HFi+5069afSZNGMGXySO7de8iP7Xtz87Pf/+w5sdfUJYv/ir2mnnfDMc41FcCpR2fOn3fDx+d+grF17dKev2aMQ6FQcPGSO40at+fK/28VZBiZ5Bu9tmg0z4CtrS0rV66kYcOGFC5cmKVLl9KiRQtu3rxJrVq1ePEi4RHWX5Ia8wxkRhlhnoGMICXzDHxLUjLPwLckJfMMfEuSM8/Aty615xmImPyz1soyGr9Ja2WlFo16BpycnOjQoQP58uVDoVDQqFEjAC5dukSpUqW0GqAQQgiR5rLY0wQaNQYmTpxI2bJlefz4Me3bt1dNRZktWzZGjRr1la2FEEKIDC6L3SbQqDEA8OOP8Qd7dO/ePUXBCCGEECLtJbkxsHDhQvr06YO+vj4LFy78Yt6BAwemODAhhBAi3cjTBAmbN28eP//8M/r6+sybl/ioYYVCIY0BIYQQmZvcJkjYw4cPE/z3x4cRUjppkBBCCCHSh8bTEa9evZqyZcuir6+Pvr4+ZcuWZdWqVdqMTQghhEgXypgYrS2ZgUYDCMePH8/cuXMZMGAA1avHTtt54cIFhgwZgp+fH5MnT9ZqkEIIIUSaktsEX7d06VJWrlxJ586dVWmtWrXCzs6OAQMGSGNACCGEyEQ0agy8f/+eypUrx0u3t7dP15ezCCGEEFqRxXoGNBoz0LVrV5YuXRovfcWKFfz8s/amcBRCCCHShTJGe0smoPGkQ6tXr+bo0aNUq1YNiJ2K2M/Pj27duuHs7KzKN3fu3MSKEEIIITKmLNYzoFFj4MaNG1SqVAmA+/dj30plZmaGmZkZN27cUOWTxw2FEEKIjE+jxsDJkye1HYcQQgiRYSilZ0AIIYTI4rJYY0DjSYeEEEII8W2QngEhhBAirkwyc6C2SGNACCGEiEtuEwghhBAiK5GeASGEECKuLNYzII0BIYQQIg6lMms1BuQ2gRBCCJHFSc+AEEIIEZfcJhBCCCGyOGkMCCGEEFmbTEcs0pVZocbpHUKGEOJ7LL1DyBAMrGundwgZwhZ9u/QOIUNo8vZceocgvlHSGBBCCCHikp4BIYQQIovLWrMRy6OFQgghRFYnPQNCCCFEHDKAUAghhMjqslhjQG4TCCGEEFmc9AwIIYQQcWWxAYTSGBBCCCHiyGpjBuQ2gRBCCJHFSc+AEEIIEZfcJhBCCCGytqx2m0AaA0IIIURcWaxnQMYMCCGEEFmc9AwIIYQQcSizWM+ANAaEEEKIuLJYY0BuEwghhBBZnPQMCCGEEHFktdsEGvUMFCpUiMmTJ+Pn56fteIQQQoj0F6PFJRPQqDEwePBg/vvvP4oUKULjxo3ZunUrkZGR2o5NCCGEEGlA48aAp6cnly9fpnTp0gwYMIB8+fLRv39/PDw8tB2jEEIIkaaUMdpbMoMUDSCsVKkSCxcuxN/fnwkTJrBq1SqqVKlChQoV+Oeff1Aqs9YMTkIIIb4N6dkYWLx4MYUKFUJfXx8HBwcuX778xfwvX76kX79+5MuXDz09PUqUKMHBgweT9ZkpGkD4/v17du3axZo1azh27BjVqlWjV69ePHnyhDFjxnD8+HE2b96cko8QQggh0lx6faPftm0bzs7OLFu2DAcHB+bPn0/Tpk3x8fHBwsIiXv6oqCgaN26MhYUFO3fuJH/+/Dx69IjcuXMn63M1agx4eHiwZs0atmzZgo6ODt26dWPevHmUKlVKladNmzZUqVJFk+KFEEKILGnu3Ln88ssvODk5AbBs2TIOHDjAP//8w6hRo+Ll/+effwgNDeX8+fPkyJEDiB3kn1wa3SaoUqUKd+/eZenSpTx9+pTZs2erNQQAChcuTKdOnTQpXgghhEhfSoXWlsjISMLDw9WWhAbdR0VF4e7uTqNGjVRpOjo6NGrUiAsXLiQY5t69e6levTr9+vXD0tKSsmXL8ueffxIdHZ2s6mrUGHjw4AGHDx+mffv2qpZIXEZGRqxZs0aT4jU2ccIwHj/y4FXYPY4c2kqxYoW/us3vv3Xn3p2LvA6/z/mz+6hSuYJqnalpbubPm8LNG6d5FXaPB/cuM2/uZExMjNXKqGxfnqOHtxES7M2zoJsc3L8JO7vvtF29JOndpwvXbp4iKMQbl5P/Usne7ov5W7dpjpvHUYJCvDl/6SCNm9RTWz9qzEDcPI7iH3SdR4892LNvPfaVy6vW16rtQNjr+wkulSqVS40qpqorntfpN2IC9Vv9TNmazXE5fT69Q0oVyT1XRo7oz4XzB3jx3Af/J178u3M1JUoUVcvTu9fPuBzbQWjIbT5EPSVXLpPUrEKK5XdqSnW3v6n7aCP2h6ZhXLHo1zcCLFrXoEHQdsqtHa6W3iBoe4JLwb6OqRF+imj7Wglf//3b2hZgxfLZ3PW5wKuwe/jcOseE8UMT/RuS3rQ5ZmD69OnkypVLbZk+fXq8zwwJCSE6OhpLS0u1dEtLSwIDAxOM88GDB+zcuZPo6GgOHjzIuHHjmDNnDlOnTk1WfTVqDNja2mqyWaoaPqwv/fv1pG//UdSo5UjEmzcc3L8JPT29RLdp374Vs2dNYMrUuVRxaIbXNW8OHtiEuXleAKytLbG2tmTkyCmUr9iQXr2H0LRpfVaumKMqw8jIkAP7N+H3+Ck1ajlSt34bXr2O4OD+TWTPnrZzOrVt14I/p4/hr+kLqVOrFTdu3GbX7rWY/b8+cVV1qMTqNfPZsG4HtWs6cmD/MTZvXUrp70qo8ty7+5DhzhOp4fA9TZt0xO/RE3btWUdeszwAXLroQfEiDmrLujXb8H3oh4fH9bSotla9ffuOksWK8MfQvukdSqrR5FypU7saS5euo2ZtR5p935kc2XNw6MBmDA0NVHkMDQ04ctSVGX8tSotqpIjFD9UpPqkbvnN24tZ4JK9vPqLC1j/IYfblBoy+jTnFJnTl5QXveOvOlv1Fbbk1aAnKmBiCD1xKrWpoJDWulfD133+pksXQ0dGhb7+R2FVowNDhE+nzS1emTYnf9f2tGT16NGFhYWrL6NGjtVJ2TEwMFhYWrFixAnt7ezp27Mgff/zBsmXLklWOQqnBkP/o6GjmzZvH9u3b8fPzIyoqSm19aGhocosku27+ZG/zucePPJg3fzlz5y0HwMTEGP8nnvTsPYTt2/cmuM35s/twu+LFoMFjAVAoFPg+cGPxkjXMnLU4wW3atWvJ+rULMcldnOjoaOwr2XHp4iEKFanCkyf+AJQtWwpPDxdKlq7J/fu+yaqHka5+svJ/zuXkv3h4XGP40Emq+nj7nGXFsvXMm7s8Xv416xZiaGhAx/a/qNKOn9jJ9eu3GDJoXIKfYWyckycBXrRq2ZVTrvG/NWfPnp3bd8+zfNl6Zv31t8Z1CfE9pvG22lK2ZnMWTB9Hwzo10i0GA+vaWi9Tk3MlLjOzPAT6X6d+g7acOav+x65uneq4HN9JXvPShIWFayXmo6Y1tVLOR/aHpvHq6n3ujPknNkGhoObVpTxZfYhHi/YkvJGOgkp7JhGw5SS5HUqTPZcR13vMSvQzyq0dTrac+nj+OEVrcTd5cS7FZaT2tTI5v/+hzr/xa59ulCiV/HPsQ9TTZG+THAG16mutrHxnTyYpX1RUFIaGhuzcuZPWrVur0rt3787Lly/Zsyf+sVm3bl1y5MjB8ePHVWmHDh3i+++/JzIyEl1d3SR9tkY9A5MmTWLu3Ll07NiRsLAwnJ2dadu2LTo6OkycOFGTIlOkcOGC5MtnicuJs6q08PBXXL58lWoO9glukyNHDipVssPlxBlVmlKpxOXEWapVS3gbgFwmxoSHv1bdj/G5c5+QkFB6OnUiR44c6Ovr49SjM9637uDr+1hLNfy6HDlyUKFiWVxPfvoDrVQqcT15nipVKya4TZWqFXE9qX5xcXE5k2j+HDly0MOpEy9fhnP9+q0E83zfoiF58uRm04adGtZEpCZNzpWEfOwCDn3xUtshpjpFjmwY2xUh9MxnPVdKJaGnr2NSuUSi2xUe+iPvQ8IJ2Pz1C3sO81zkbVSRgM0ntBGy1qTltTIpcuUyybDHUHo8Wqirq4u9vT0uLi6qtJiYGFxcXKhevXqC29SsWZN79+4RE/Ppg+7cuUO+fPmS3BAADRsDmzZtYuXKlQwdOpTs2bPTuXNnVq1axfjx47l48aImRaaIlWXs4xZBQc/U0oOCQ7Cyiv8oBsR+s8mePTvBQSFq6cHBz7CyNE9wm7x5TfljzGBWrd6kSnv9OoKGjX/kp85teR1+n7AXd2jatB4tHbskewBHSuTNaxpbn2D1+jwLDsEykfpYWpoR/Oz5V/M3bVafp4HXCH7uTd/+TrRp1Y3Q5y8SLLNrtw64HD+Dv3/C97dE+tLkXIlLoVAwd/Ykzp27zM2bPlqPMbXlyGOCTvZsRD17qZYe9ewluha5E9wmV9WS5PupAbeHxu9hS0i+DnWJfv2OZwe+/Hx4Wkura2VSFC1aiH59nVi5cqPGZXyLnJ2dWblyJevWrePWrVv8/vvvREREqJ4u6Natm9otht9//53Q0FAGDRrEnTt3OHDgAH/++Sf9+vVL1udq1BgIDAykXLnYwWE5c+YkLCwMgJYtW3LgwIGvbp/QyMrk3K3o3LkNL0PvqJYcOVL/3ryxcU727VnPrVt3mDT505gBfX19Vi6fzfkLV6hZy5E6dVtz86YPe/esR19f8y7/jOTM6YvUruFI44btcTl2mrXrFyU4DsHa2oqGjWqzYf32dIhSJCQ1zpVFC/+kTJmS/NTl2x1X8blsRvp8t3gAt4cu533oqyRtk69zfQL/O0NM5PtUju7L0uNamRTW1lYc2LeRnf/uZ/U/GXMuGqVSobUlOTp27Mjs2bMZP348FSpUwNPTk8OHD6sGFfr5+REQEKDKb2Njw5EjR3Bzc8POzo6BAwcyaNCgBB9D/BKNjowCBQoQEBBAwYIFKVq0KEePHqVSpUq4ubl9cRDKR9OnT2fSpElqaQqdnCiyJW308b59R7l8+arqZz292K4QS0tzAgODVemWFmZ4et1MsIyQkFA+fPiAhaWZWrqFhTmBcVrNOXMacXD/Jl69iqBd+958+PBBta5zp9bY2tpQs3YrVYOmS9d+hAR706pVkyTfg02p589fxNbHQr0+5hZm8b4FfBQUFIJFnD/qCeV/8+YtDx484sGDR1xx88TD04Vu3dozd476AJWfu/5IaOhLDh5wQWQM2jhXPrdg/lRafN+I+g3b8vRpwFfzZ0TvQ8OJ+RCNrnlutXRd89xEBb+Ml9+gkCUGBS2w2zBSlabQib3A13u6hUs1BvP2UZBqXS6HUhgVz8/NPvNTI/xkSetrZVLky2fJ8WM7uHDRnd9+H5Hs7dNKek4j3L9/f/r375/gOldX13hp1atXT3GvvEY9A23atFHd0xgwYADjxo2jePHidOvWjZ49e351+4RGVip0jL+63UevX0dw/76vavH2vkNAQBAN6tdS5TE2zknVqhW5eMk9wTLev3+Ph8c1tW0UCgUN6tfi4sVP2xgb5+TwwS1ERUXRum2PeM+GGhoaEBMTo9az8fFnHZ0UzfacLO/fv8fz6g3q1vs0EEehUFC3XnXcPrsYfM7t8lW1/AD169dKNP9HOjoKdPXi34vq0qUdWzfvUmssifSljXPlowXzp9L6h2Y0btohTcfDaJvyfTSvrj3AtHbZT4kKBaa1yxJ+5U68/G/u+XOp7lDcGo5QLSFH3Hlx7iZuDUfwzl+9+9z6pwaEe97ntfej1K7KV6XltTIprK2tcDm+Ew+Pa/TqPUSmrM9ANOoZmDFjhurfHTt2pGDBgly4cIHixYvj6Pj1Z2r19PTi9SAoFMnrSolr4aJVjBk9kLv3HuDr+5hJE4fj7x/Enj1HVHmOHt7G7j2HWLJ0LQDzFqxkzep5uHtcw83tKgMH/IKRkQFr120DPjUEDAz16dZjACYmxqo5Bp49e05MTAzHXU7z14yxLFr4J4uX/IOOjg4jhvfnw4cPuCYw2j41Lf77H5Yun8VVj+u4u3vRt58TRoaGbNwYO5hv2YrZBPgHMmnibACWLlnLwcOb6T+gF0eOnKTdjy2pWKksgwb+AcQ2dIYN78vBgy4EBQaTN28eevfpQj5rK3bvOqT22XXr1aBQ4YKs+/++y6zevHmL3/+fCgF46h/E7Tv3yWViTL4k3lPP6DQ5VxYt/JPOnVrTtl1PXr16rRpXEhb2infv3gGx3zatrCwoWrQQAOXKluLV6wj8/J7yIoMNEnu8bD+lF/bjlecDwq/ew6bP92Qz1MN/qysApRf1IzIwlAfTthAT+Z6I2+qNnw9hEQDx0rPlNMCiVTXuTtiQJvXQRGpcK+Hrv39raytcju3Ez+8JI0ZOUXssMbHey/SkjEnZ36TMRis3kKpXr57oSMe0Mmv2EoyMDFm2ZCa5c5tw7pwbLRy7qH2TL1LEFrP/Px8PsGPHXszN8jBx/DCsrMzx8rpJi5ZdVIPwKlUsh4NDJQDu3Fb/w160uAOPHj3Bx+c+rdv0YNxYZ86e3ktMTAyenrHlfN4Nlxb++/cAec3yMGbsYCwtzbh+7RZt2zjxLDh2kGABm3xqI04vX/Kgd88hjB3nzPiJQ7l//xE/dfqdW96x346io6MpUbIonX9uS968poSGvsTD/RrNm3Tk9q27ap/dtVt7Ll5w5+6dB2lX4VRw4/Zdeg741B08c9EKAH5o3ohpY4emV1hapcm58vtv3QE44fKvWlk9ew1h/YbYMSK/9unK+HGf9pHryV3x8mQUwXsukCOvCUVGdEDXIjevbvri1flP3j+LHf+kn98MYpL/rdWyTQ1AQdCus1/Nm15S41oJX//9N2pYh+LFC1O8eGH8fNV7FFL6aHlqyGqdFkmeZ2Dv3qTf+27VqlWyA8mIB0N6SMk8A9+SjDDPQEaQGvMMZEbanmcgs9LGPAPfitSeZ+BRpUZfz5REth7Hv54pnSW5Z+DzCRAgtls/bjviY1d/Wj5SJ4QQQoiUSfIIt5iYGNVy9OhRKlSowKFDh3j58iUvX77k0KFDVKpUicOHD6dmvEIIIUSqU8YotLZkBhqNGRg8eDDLli2jVq1Po0ubNm2KoaEhffr04dathGenE0IIITKDrDZmQKNn3+7fv0/u3LnjpefKlQtfX98UhiSEEEKItKRRY6BKlSo4OzsTFPRpoo2goCCGDx9O1apVtRacEEIIkR7kNkES/PPPP7Rp04aCBQtiY2MDxE6RWKJECXbt2qXVAIUQQoi0ltxphDM7jRoDxYoV49q1axw/flw1PqB06dI0atQoxZMHCSGEECJtaTzp0IkTJzh58iTBwcH/n2jHky1btgCxPQdCCCFEZpWe7yZIDxo1BiZNmsTkyZOpXLky+fLlk94AIYQQ35QYuU3wdcuWLWPt2rV07dpV2/EIIYQQIo1p1BiIioqiRo0aX88ohBBCZEJZbQChRo8W9u7dm82bN2s7FiGEECJDkEcLk+Ddu3esWLGC48ePY2dnR44cOdTWz507VyvBCSGEEOkhq81AqFFj4Nq1a1SoUAGAGzduqK2TwYRCCCFE5qJRY+DkyZPajkMIIYTIMDJL9762aDzPgBBCCPGtymqPFmo0gFAIIYQQ3w7pGRBCCCHiyGqPFkpjQAghhIgjqz1NILcJhBBCiCxOegaEEEKIOLLaAEJpDAghhBBxZLUxAxrdJjh9+jQfPnyIl/7hwwdOnz6d4qCEEEIIkXY0agzUr1+f0NDQeOlhYWHUr18/xUEJIYQQ6Ump1N6SGWh0m0CpVCY47fDz588xMjJKcVBCCCFEepIxA1/Qtm1bIPb9Az169EBPT0+1Ljo6mmvXrmn8auNsOvJgA8C7D1HpHUKGYGBdO71DyBDe+p9J7xAyBDkeRFrLamMGktUYyJUrFxDbM2BsbIyBgYFqna6uLtWqVeOXX37RboRCCCGESFXJagysWbMGgEKFCjFs2DC5JSCEEOKbJLcJkmDChAnajkMIIYTIMDLJuD+t0XiegZ07d7J9+3b8/PyIilK/z+3h4ZHiwIQQQgiRNjQatbdw4UKcnJywtLTk6tWrVK1albx58/LgwQOaN2+u7RiFEEKINBWjVGhtyQw0agwsWbKEFStWsGjRInR1dRkxYgTHjh1j4MCBhIWFaTtGIYQQIk0plQqtLZmBRo0BPz8/1SOEBgYGvHr1CoCuXbuyZcsW7UUnhBBCiFSnUWPAyspKNQNhwYIFuXjxIgAPHz5EmVmmWxJCCCESEaPFJTPQqDHQoEED9u7dC4CTkxNDhgyhcePGdOzYkTZt2mg1QCGEECKtKVFobckMNHqaYMWKFcTExLZ3+vXrR968eTl//jytWrXi119/1WqAQgghhEhdGjUGdHR00Pls+uBOnTrRqVMnrQUlhBBCpKeYLHbHW6PGQJ06dahXrx5169alZs2a6OvrazsuIYQQIt3EZJLufW3RaMxAkyZNuHjxIj/88AO5c+emVq1ajB07lmPHjvHmzRttxyiEEEKkKRkzkARjx44F4MOHD7i5uXHq1ClcXV2ZOXMmOjo6vHv3TqtBCiGEECL1aDwdMcCDBw+4fv06Xl5eXLt2DWNjY+rUqaOt2IQQQoh0kVkeCdQWjRoDP/30E6dOnSIyMpI6depQt25dRo0ahZ2dHQpF5ugSEUIIIRKTWbr3tUWjxsDWrVsxMzOjd+/eNGjQgFq1amFoaKjt2IQQQgiRBjQaQPj8+XNWrVpFVFQUo0ePxszMjBo1ajBmzBiOHj2q7RiFEEKINJXVZiBUKLUwf/C9e/eYOnUqmzZtIiYmhujo6GSXoadvk9IwxDckOiaznEKp663/mfQOIUMwsK6d3iGIDOZD1NNULf+gpfbmzvk+aKvWykotGt0meP78ueoJAldXV7y9vcmdOzeOjo7UrVtX2zEKIYQQIhVp1BiwsLDAzMyM2rVr88svv1CvXj3KlSun7diEEEKIdCEDCJPg2rVrlClTRtuxCCGEEBlCTNZqC2g2gFAaAkIIIcS3Q+NJh3bu3Mn27dvx8/MjKipKbZ2Hh0eKAxNCCCHSi7ybIAkWLlyIk5MTlpaWXL16lapVq5I3b14ePHhA8+bNtR2jEEIIkaaUWlwyA40aA0uWLGHFihUsWrQIXV1dRowYwbFjxxg4cCBhYWHajjFJfvu1Oz4+5wl7eZczp/dSuXKFL+Zv27YF17xOEvbyLu5XjtGsaX219T/80IwD+zfh//Qake8eY2f3Xbwyjh7dTuS7x2rL34v+1Ga1ki099sPn9u5ZT+S7x7RybJrSqqSKiROG8fiRB6/C7nHk0FaKFSv8xfwjR/TnwvkDvHjug/8TL/7duZoSJYqq5end62dcju0gNOQ2H6KekiuXSWpWIU1c8bxOvxETqN/qZ8rWbI7L6fPpHVKqSO7x8LkRw/vxIeopc2ZPUqXZ2hbgQ9TTBJd27VqmRhWSRJN6/v5bd+7ducjr8PucP7uPKnGuJXp6eixcMI2ggBu8DL3D9m0rsLAwS7CsPHlM8X1wJcHzo3PnNrhfOUb4y3s8fuTByhVzyJPHVOO6aktWm2dAo8aAn58fNWrUAMDAwIBXr14B0LVrV7Zs2aK96JLoxx8dmTlzHNOmzceh2vdcv+7N/n0bMDfPm2D+atXs2bD+b9au3YqDQ3P27jvCjh2r+O67kqo8RkaGnDt/mT/GfvmP++rVmyhoW0m1jB6Tfo2B9NwPAAMH9EYL01akmuHD+tK/X0/69h9FjVqORLx5w8H9m9DT00t0mzq1q7F06Tpq1nak2fedyZE9B4cObMbQ0ECVx9DQgCNHXZnx16K0qEaaePv2HSWLFeGPoX3TO5RUo8nx8FFl+/L80rsLXte81dIfP/Ynv00FtWXipFm8evWaw4dPpFZVvkiTerZv34rZsyYwZepcqjg0w+uaNwcPbFK7lsyZPZGWLRrTqfOvNGjYDut8VuzcvirB8laumM31697x0mtUr8zafxawZs0W7CrUp1PnX6lSpQLLl81MecVFsmjUGLCysiI0NBSAggULcvHiRQAePnyYLn8MBg38hX/+2cL69du5ffsu/fqP5s2bd3Tv3jHB/P379eLoUVfmzlvObZ97TJo0m6tXb9D39+6qPJs3/8effy7gxImzX/zsN2/eEhT0TLW8evVaq3VLjvTcD3Z23zFoUB/6/DpMq3XSpoEDevPn9AXs23eU69dv0cNpENbWlvzwQ+K9GC0cu7B+w3a8ve9w7Zo3PXsPxta2APaV7FR5Fi5axcxZi7l06dsZK1O7ehUG9ulOo7o10zuUVKPJ8QCxDeT16//mt99H8PLFS7V1MTExateDoKBn/PBDc3bs3EdERPq83l2Teg4Z9AurVm9m3frt3Lp1l779RvHmzVucesROxGNiYkxPp04MGzGJk67n8Lh6nV6/DKFGjSo4VK2kVtavfbqRO5cJc+ctj/c51arZ4+v7mL8X/4Ov72POnXdj5cqNVKlcUbs7QQMxCoXWlsxAo8ZAgwYN2Lt3LwBOTk4MGTKExo0b07FjR9q0aaPVAL8mR44cVKpUTu2PlVKp5MTJM1RzsE9wG4dqleL9cTt2/BQOieT/kk6d2vD0iRce7seZMmUkBgb6yS5DG9JzPxgY6LN+3SIGDx5LUNCz5AefBgoXLki+fJa4fFbf8PBXXL58NdH9k5CPXZyhcf4IiMwlJcfDooV/cuigCy4nvj47ZKWK5ahYoSxr1qTPDHSa1DP2WmKnVj+lUonLibNUqxa7jX0lO3R1dXFx+ZTHx+c+jx49UeUBKF26OGP/GEyPnoOISWBW0YsX3bGxsaZ5swYAWFiY0a5tCw4ddklZxbUgq40Z0OhpghUrVqh+sf369SNv3rycP3+eVq1a8euvv351+8jISCIjI9XSlEqlRm88NDPLQ/bs2QkKVv8jFBwUQskSxRLcxsrSnKDgkHj5LS3Nk/XZ27btxu/RU/wDgihXrhTTpo6hRPGidOzUJ3mV0IL03A+zZ03gwkV39u3PuO+lsLK0AIjXWAkKDsHKyiJJZSgUCubOnsS5c5e5edNH6zGKtKPp8dChQysqVixLteotkvQ5Tk6d8b51hwsXr2gebApoUs+P15LgoDjXhuBnlCoZO17G0sqcyMhIwsLC4+Wxsoq9fujq6rJxwxJGjp7K48f+FClsG++zzl+4QtfuA9i8aSn6+nrkyJGDffuPMmDgH5pVWGgs2T0DHz58YOrUqQQGBqrSOnXqxMKFCxkwYAC6urpfLWP69OnkypVLbYmODv/qdhnN6tWbOXb8FDdv3mbr1t306j2Y1q2bU6RI/IP+W9WyRWPq1avJsGET0zsUNZ07t+Fl6B3VkiOHxk/Rqixa+CdlypTkpy7f7n30b5U2jocCBayZN2cy3boPiPdlJiH6+vp07tQ6TXsFUuO419SfU0dz+/ZdNm/+L9E8pUsXZ96cSUydNo+q1ZrzfYufsLUtwJLFM9Iw0oRltQGEyT5SsmfPzsyZM+nWrZvGHzp69GicnZ3V0szMvzxKPTEhIaF8+PABSwv1b7MWlmaJdlkHBj3DMs6o1y/lT6rLl68CULRIIR48eJSispIrvfZDvXo1KFLEluCgm2rpW7cu5+y5yzRp0iHJZWnTvn1HVb8PAD292EaqpaU5gYHBqnRLCzM8vW7G2z6uBfOn0uL7RtRv2JanTwO0H7BIVdo4HipVKoelpTlulw6r0rJnz07t2tXo17cHhjkLq3WFt2vXAkNDAzZs3KHt6iRKG/X8eC2xsIxzbbAwJ/D/14agwGfo6emRK5eJWu+AhYU5gYGxeerVr0m5sqVo1za2F+Vjz29QwHWmz1jIpMlzGDliAOcvXGHO3GUAXL9+i4iIN5xy3c34CTPVYk5rMgNhEjRs2JBTp05p/KF6enqYmJioLZrcIgB4//49Hh7XqV//00AnhUJB/Xq1uHjJPcFtLl30UMsP0LBBbS4lkj+pypePnZkxIDAoReVoIr32w6zZS7Cv3IQqVZupFoDhwyfRp89QDWqiHa9fR3D/vq9q8fa+Q0BAEA3q11LlMTbOSdWqFRPdPx8tmD+V1j80o3HTDvj6Pk7t0EUq0MbxcOLEWcpXbIB9lSaqxe2KJ5u37MK+SpN498R79ujEvv3HCAkJTdW6fU4b9Yy9llxT20ahUNCgfi0uXozdxt3jGlFRUTRo8ClPiRJFsbUtoMrToeMvVKrcWLWvPg4urle/LUuWrgXA0FA/3n6Ljo5RfaZIOxr1ITVv3pxRo0Zx/fp17O3tMTIyUlvfqlUrrQSXVAsWrmT1qrm4e1zjipsnAwb0wsjIgPXrtwOwevU8/P0DGTfuLwD+Xrya48d2MHhQHw4dcqF9h1bY29vRt98oVZmmprmxsbHGOp8lgOrZ8o8jhIsUsaVjx9YcPnyC0NAXlCtbmlmzJnD6zEVu3LidpvX/KD32w8clrseP/TPcH86Fi1YxZvRA7t57gK/vYyZNHI6/fxB79hxR5Tl6eBu79xxSXawWLfyTzp1a07ZdT169eq0aTxEW9op3794Bsd+6rKwsKFq0EADlypbi1esI/Pye8iKTDjR88+Ytfk/8VT8/9Q/i9p375DIxJl8Sx1hkdMk9Hl6/jog3VuRNxBueP38RL71o0ULUrl0Nx1Zd06QuX6LJcT9vwUrWrJ6Hu8c13NyuMnDALxgZGbB23TYgdhDiP2u2MnvmBF6EviQ8/BUL5k/lwoUrXLoc+1RN3N5Rs7x5ALh1+66qN2H//uMsXzaTX/t04+gxV/JZWTBnziQuX/YgICDtv1R9Lj1nIFy8eDGzZs0iMDCQ8uXLs2jRIqpWrfrV7bZu3Urnzp354Ycf2L17d7I+U6PGQN++sfdM586dG2+dQqEgOjpak2I1tnPnPszN8jB+/FCsLM3x8vLGsVVXgv8/OM7GJj8xMZ/GdF686E637gOYNHE4kyeP4N49X9q3742396cTumXLxqxa+al+mzYuAWDK1LlMnTpP1Soe0D/2D+6TJwHs2nWQ6TMWplGt40uP/ZCZzJq9BCMjQ5YtmUnu3CacO+dGC8cuavd/ixSxxcwsj+rn33+LfczyhMu/amX17DWE9RtiG1m/9unK+HGfekFcT+6KlyezuXH7Lj0HjFT9PHPRCgB+aN6IaWPTr8dHmzQ5HpLKqUcnnjwJ4OgxzXtQtUWTeu7YsRdzszxMHD8MKytzvLxu0qJlF9W1BGDosInExMSwfdsK9PT0OHrMlf4DxiQrtvUbtmNsbETfvj2YNXM8L1+GcdL1XLrO1/JRej0FsG3bNpydnVm2bBkODg7Mnz+fpk2b4uPjg4VF4g1xX19fhg0bRu3atTX6XIUyg8wSo6dvk94hiAwkOoHHkLKit/5ff3wtKzCw1uwCJ75dH6Kepmr5G627aK2s9g9Xxxt0qqenl+DETw4ODlSpUoW///4biJ27wsbGhgEDBjBq1Kh4+QGio6OpU6cOPXv25MyZM7x8+TLZPQMajRlIyMuXL7VVlBBCCJGuYhTaWxJ6gm769OnxPjMqKgp3d3caNWqkStPR0aFRo0ZcuHAh0VgnT56MhYUFvXr10ri+GjUG/vrrL7Zt26b6uX379uTJk4f8+fPj5eWlcTBCCCFERqDNRwtHjx5NWFiY2jJ69Oh4nxkSEkJ0dDSWlpZq6ZaWlmqP83/u7NmzrF69mpUrV6aovho1BpYtW4aNTWy3/rFjxzh+/DiHDx+mefPmDB8+PEUBCSGEEOlNmzMQJvQEXVLegfE1r169omvXrqxcuRIzs4RfEpVUGg0gDAwMVDUG9u/fT4cOHWjSpAmFChXCwcEhRQEJIYQQWZGZmRnZsmUjKEj9SYqgoCCsrKzi5b9//z6+vr44Ojqq0j4+qpk9e3Z8fHwoWrRovO0SolHPgKmpKY8fxz42dvjwYdX9DaVSmeZPEgghhBDaps0xA0mlq6uLvb09Li6f3s0QExODi4sL1atXj5e/VKlSXL9+HU9PT9XSqlUr6tevj6enp+pLe1Jo1DPQtm1bfvrpJ4oXL87z589p3rw5AFevXqVYsYTnwRdCCCEyi/R6nsnZ2Znu3btTuXJlqlatyvz584mIiMDJyQmAbt26kT9/fqZPn46+vj5ly5ZV2z537twA8dK/RqPGwLx58yhUqBCPHz9m5syZ5MyZE4CAgADVHARCCCGESJ6OHTvy7Nkzxo8fT2BgIBUqVODw4cOqQYV+fn7o6GjtQUAVmWdAZEgyz0AsmWcglswzIOJK7XkGlhfQ3jwDvz7ZqLWyUkuKXmnl7e2Nn58fUVFRaulpPR2xEEIIoU3KLPZqBI0aAw8ePKBNmzZcv34dhULBx86Fjy+WkEGEQgghROah0Y2HQYMGUbhwYYKDgzE0NOTmzZucPn2aypUr4+rqquUQhRBCiLSlzUmHMgONegYuXLjAiRMnMDMzQ0dHBx0dHWrVqsX06dMZOHAgV69e/XohQgghRAaVWf6Ia4tGPQPR0dEYGxsDsZMk+PvHvurU1tYWHx+fL20qhBBCiAxGo56BsmXL4uXlReHChXFwcGDmzJno6uqyYsUKihQpou0YhRBCiDSVIR6zS0MaNQbGjh1LREQEEPu2pJYtW1K7dm3y5s2r9gIjIYQQIjNKzsyB3wKNGgNNmzZV/btYsWLcvn2b0NBQTE1NVU8UCCGEEJlVVhszkKJ5Bj6XJ08ebRUlhBBCiDSU5MZA27Ztk1zof//9p1EwQgghREYgPQOJyJUrV2rGIYQQQmQYMoAwEWvWrEnNOIQQQgiRTjR+9dGHDx84fvw4y5cv59WrVwD4+/vz+vVrrQUnhBBCpIcYhfaWzECjAYSPHj2iWbNm+Pn5ERkZSePGjTE2Nuavv/4iMjKSZcuWaTtOIYQQIs1ktTEDGr+boHLlyrx48QIDAwNVeps2bXBxcdFacEIIIYRIfRr1DJw5c4bz58+jq6urll6oUCGePk3dd0wLIYQQqU0GECZBTExMgq8pfvLkieqdBUIIIURmFZPFmgMaNQYaN27M/PnzWbFiBQAKhYLXr18zYcIEvv/+e40CiY7JandoEpbXQBpTAFv07dI7hAzBwLp2eoeQIbz1P5PeIWQIcjyI1KJRY2Du3Lk0bdqU7777jnfv3vHTTz9x9+5d8ubNy5YtW7QdoxBCCJGmstrXU40GEJ45cwYvLy/GjBnDkCFDqFixIjNmzMDT05NZs2ZpO0YhhBAiTSm1uGQGGvUM/P777+TOnZsuXbqopTs7O7NlyxZpEAghhMjUpGcgCTZt2kTnzp05e/asKm3gwIFs3bqVkydPai04IYQQQqQ+jXoGWrRowZIlS2jVqhXHjh1j9erV7NmzB1dXV0qUKKHtGIUQQog0lVlmDtQWjV9h/NNPP/Hy5Utq1qyJubk5p06dolixYtqMTQghhEgX8mhhIpydnRNMNzc3p1KlSixZskSVNnfu3JRHJoQQQog0keTGwNWrVxNML1asGOHh4ar1CkUW61sRQgjxzcla/QLJaAzIwEAhhBBZhTxNIIQQQogsReMBhEIIIcS3SgYQCiGEEFlc1moKyG0CIYQQIsuTngEhhBAijqw2gFAaA0IIIUQcMmZACCGEyOKyVlNAw8ZAdHQ08+bNY/v27fj5+REVFaW2PjQ0VCvBCSGEECL1aTSAcNKkScydO5eOHTsSFhaGs7Mzbdu2RUdHh4kTJ2o5RCGEECJtxWhxyQw0foXxypUrGTp0KNmzZ6dz586sWrWK8ePHc/HiRW3HKIQQQqQppRb/yww0agwEBgZSrlw5AHLmzElYWBgALVu25MCBA9qLTgghhBCpTqPGQIECBQgICACgaNGiHD16FAA3Nzf09PS0F50QQgiRDuQ2QRK0adMGFxcXAAYMGMC4ceMoXrw43bp1o2fPnloNUAghhEhrMSi1tmQGGj1NMGPGDNW/O3bsiK2tLefPn6d48eI4OjpqLTghhBBCpD6NGgMREREYGRmpfq5WrRrVqlXTWlBCCCFEesoc3+e1R6PbBJaWlvTs2ZOzZ89qOx4hhBAi3WW12wQaNQY2btxIaGgoDRo0oESJEsyYMQN/f39tx5ZsEycM4/EjD16F3ePIoa0UK1b4q9v8/lt37t25yOvw+5w/u48qlSuo1tnaFuBD1NMEl3btWqqV061rBzzcj/E6/D7+T7xYuGCatquXJE69f8LtmguPgrw45LKNipXKfTG/Y+umnHU7yKMgL1zP76Vh4zpq6xcsmU5Q2G21Zcu/K+OV06hJXQ65bMM30BOfR5dYu+lvrdZLG/I7NaW629/UfbQR+0PTMK5YNEnbWbSuQYOg7ZRbO1wtvUHQ9gSXgn0z/q0yTc6Vj0YM78eHqKfMmT1JlZaccyUzueJ5nX4jJlC/1c+Urdkcl9Pn0zskrdD2tdLUNDfz503h5o3TvAq7x4N7l5k3dzImJsaqPHnymHJg30b8fN2JePWAh/fdWDB/KsbGOVOjiiKZNGoMtG7dmt27d/P06VN+++03Nm/ejK2tLS1btuS///7jw4cP2o7zq4YP60v/fj3p238UNWo5EvHmDQf3b/ri0w3t27di9qwJTJk6lyoOzfC65s3BA5swN88LwOPH/uS3qaC2TJw0i1evXnP48AlVOYMH9WHK5BHMnLUYuwoNaNq8E0ePuaZ2leP5oW1zJv05ijl/LaZxnbbcvOHD1l2rMDPLk2D+ylUrsmz1HDZv2Emj2m04dOA4azf/TanSxdXyuRw7TdnitVTLb72Gqq1v0aoJf6/4iy2b/qNBzdY4NvmJ/3buT7V6asLih+oUn9QN3zk7cWs8ktc3H1Fh6x/kMDP54nb6NuYUm9CVlxe84607W/YXteXWoCUoY2IIPnAptaqhFZqcKx9Vti/PL7274HVNfX8k9VzJbN6+fUfJYkX4Y2jf9A5Fa1LjWmltbYm1tSUjR06hfMWG9Oo9hKZN67NyxRxVGTExMezdd5Q2bZ0oXaY2vXoPoWHD2ixZPCOxj01XWe1pAoVSqdRKH8aiRYsYPnw4UVFRmJmZ8dtvvzFq1CgMDQ2TtH123fwp+vzHjzyYN385c+ctB8DExBj/J5707D2E7dv3JrjN+bP7cLvixaDBYwFQKBT4PnBj8ZI1zJy1OMFt3C4f4erV6/T5dRgAuXPnws/XndZtenDiZMpvm+Q1MP56pkQcctnGVY8bjBk+BYitz1VvV1av2MiiefG/za9YMxdDQ0O6dPxNlXbw+FZuXL/NiCETgdiegVy5jOnxc/8EPzNbtmxcue7CrOmL2LzhX41jj2uLvp3WygKwPzSNV1fvc2fMP7EJCgU1ry7lyepDPFq0J+GNdBRU2jOJgC0nye1Qmuy5jLjeY1ain1Fu7XCy5dTH88cpWou7yYtzWivrI03OFQAjI0PcLh9hwIAxjBk9EE8vb4YOm5Bo/rjnSkq89T+T4jJSqmzN5iyYPo6GdWqkWwwG1rVTXEZaXSvbtWvJ+rULMcldnOjo6ATz9O/Xk6HOv1O4aJVk1+ND1NNkb5McvQv9qLWyVvnu1FpZqUWjnoGPgoKCmDlzJt999x2jRo3ixx9/xMXFhTlz5vDff//RunVrLYX5ZYULFyRfPktcTnz6Yxwe/orLl69SzcE+wW1y5MhBpUp2uJz4dJFRKpW4nDhLtWoJb1OpYjkqVijLmjVbVWmNGtVBR0eBdX4rrl9zxffBFbZsXkaBAtZaql3S5MiRA7sKZTjj+qkbU6lUctr1ApWrVEhwG/sqFTjtqt7tedLlXLz8NWpV5ea9c5y7coi/5k7A1DS3ap1d+e+wzm9FTIyS42f+45rPaTbvXBGvdyE9KXJkw9iuCKFnrn9KVCoJPX0dk8olEt2u8NAfeR8STsDmk1/9jBzmucjbqCIBmzP2t2BNzpWPFi38k0MHXdTOmcQkdK6I9JdW10qAXCbGhIe/TrQhkC+fJW1aN+f0mQsa1iZ1ZbWeAY2eJvjvv/9Ys2YNR44c4bvvvqNv37506dKF3Llzq/LUqFGD0qVLJ7h9ZGQkkZGRamlKpRKFQqFJOFhZWgAQFPRMLT0oOAQrK4sEtzEzy0P27NkJDgpRSw8OfkapkgnfS3Zy6oz3rTtcuHhFlVakcEF0dHQYNXIAzs7jCQt7xeRJIzh8aAsVKzXi/fv3GtUpufLkNSV79uw8C36ulv7sWQjFSyR8P9DC0izB/BaWZqqfT7qc4eC+o/g9ekqhwjaMHj+ELf+u4PtGnYiJicG2sA0Aw0b1Y8Iff/HY7ym/93fivwPrqWHfjJcvwrRc0+TLkccEnezZiHr2Ui096tlLDIsn3GjLVbUk+X5qgFvDEUn6jHwd6hL9+h3PDlxOabipSpNzBaBDh1ZUrFiWatVbJOlzEjpXRPpLq2tl3rym/DFmMKtWb4q3buOGxbRybIqhoQH79h+lz6/DEyhBpDWNegacnJywtrbm3LlzeHp60r9/f7WGAIC1tTV//PFHgttPnz6dXLlyqS3KmFdJ/vzOndvwMvSOasmRI/XfxKyvr0/nTq3jfdPR0dFBV1eXIUPGcfTYKS5d9uDnrn0pXqww9eulX3eituz+9yBHDp3klvcdDh1woWvH36hob0fN2lUB0FHEHkIL5iznwN6jXPO8yaC+o1EqlTi2bpaeoWssm5E+3y0ewO2hy3kfmrTjMl/n+gT+d4aYyLRp/CWVNs6VAgWsmTdnMt26D4jXiE9IYueKSHvpca00Ns7Jvj3ruXXrDpMmz4m3fuiwiVRxaErrtj0oUsSW2bMSv9WUnrLauwk0OjICAgK+OhbAwMCACRMS/iWPHj0aZ2dntTTTvKWS/Pn79h3l8uWrqp/19HQBsLQ0JzAwWJVuaWGGp9fNBMsICQnlw4cPat+CASwszAmM02oGaNeuBYaGBmzYuEMtPSAgCADvW3fVyg4JCcXGJmXjIJIj9PkLPnz4gLlFXrV0c3OzeC36j4KDQpKVH+CR7xNCQkIpVMSWM6cuqr5h+Ny+p8oTFfUeP9/HFCiQT9PqaNX70HBiPkSja55bLV3XPDdRwS/j5TcoZIlBQQvsNoxUpSl0Ynut6j3dwqUag3n7KEi1LpdDKYyK5+dmn/mpEX6KaONcqVSpHJaW5rhdOqxKy549O7VrV6Nf3x4Y5ixMTMynztDEzhWR9tL6WpkzpxEH92/i1asI2rXvneBg8qCgZwQFPcPH5z4vQl9yynU30/6crxZPRpBZuve1RaOegc8bAu/evSM8PFxt+Ro9PT1MTEzUluTcInj9OoL7931Vi7f3HQICgmhQv5Yqj7FxTqpWrcjFS+4JlvH+/Xs8PK6pbaNQKGhQvxYXL8bfpmePTuzbf4yQkFC19PMXYrtBS5b41F1mapobM7M8+Pk9SXKdUur9+/dc87xJ7brVVWkKhYLadatxxc0zwW3c3TzV8gPUrV8j0fwA+awtyZMnN8H/P3G9PG/w7l0kxYp/uhWRPXt2bArm58nj9H/cFED5PppX1x5gWrvsp0SFAtPaZQm/cide/jf3/LlUdyhuDUeolpAj7rw4dxO3hiN456/eWLL+qQHhnvd57f0otauSbNo4V06cOEv5ig2wr9JEtbhd8WTzll3YV2mi1hCAxM8VkfbS8lppbJyTwwe3EBUVReu2PZLUi6SjE/sn6GMjRaQfjWcgHDlyJNu3b+f58+fx1ic2YCQ1LVy0ijGjB3L33gN8fR8zaeJw/P2D2LPniCrP0cPb2L3nEEuWrgVg3oKVrFk9D3ePa7i5XWXggF8wMjJg7bptamUXLVqI2rWr4diqa7zPvXv3AXv2Hmbu3En8/vtIwl+9YtrU0dz2ucdJ17R9JnnZ4rUsXDoDz6s3uOp+jT59u2NoZMDWjf8BsGjZDAIDgpk2aS4AK5ZuYPfB9fzW34njR1xp3a4F5SuWYdig8QAYGhkybFQ/Duw5SnBwCIUK2zBu8nAePvDjpEvsAKTXryJY/89Who8ewNOngTzx86ffoNj3U+zdfTiBKNPH42X7Kb2wH688HxB+9R42fb4nm6Ee/ltdASi9qB+RgaE8mLaFmMj3RNx+rLb9h7AIgHjp2XIaYNGqGncnbEiTemhDcs+V168juHnTR62MNxFveP78Rbz0L50rmdGbN2/xe/KpUfvUP4jbd+6Ty8SYfF8YY5GRpca18mNDwMBQn249BmBiYqyaY+DZs+fExMTQvFkDLCzMuOLuxevXEZT5riQzZozl3LnLPHqUdl+ckipGOw/aZRoaNQZGjBjByZMnWbp0KV27dmXx4sU8ffqU5cuXq723IC3Nmr0EIyNDli2ZSe7cJpw750YLxy5qrdMiRWzVnrnfsWMv5mZ5mDh+GFZW5nh53aRFyy4EB6t/83Pq0YknTwI4euxUgp/dw2kQc2ZPZO+edcTEKDl95gItWnZJ8/kW9vx3iLx58zBizAAsLM25ef0Wndv+wrNnsQ22/AWsiYn5dIBfuXyV33sPY9TYwYwZP4SH933p8VN/bv//lkdMdDTflSlJx86tMcllTGDAM06dPMdfUxcQFfXp3vikcbP4EB3N4uV/oa+vj4e7F+0cexD28uu9RGkleM8FcuQ1ociIDuha5ObVTV+8Ov/J+2exAxz185tBTPJPfss2NQAFQbsyz2ycmpwrSfW1cyWzuXH7Lj0HfLpdNHPRCgB+aN6IaWOHJrZZhpYa18pKFcvh4FAJgDu31b8EFS3uwKNHT3j79h29e/3MnNkT0dPT5fGTAHbvPshfMxN+NDG9Za2mgIbzDBQsWJD169dTr149TExM8PDwoFixYmzYsIEtW7Zw8ODBZAeS0nkGvhUpmWfgW6LteQYyq9SYZyAzygjzDGQE2phn4FuR2vMMdLFtq7WyNj76T2tlpRaNxgyEhoZSpEgRAExMTAgNjb03WKtWLU6fPq296IQQQoh0IO8mSIIiRYrw8OFDAEqVKsX27dsB2LdvX7xHDIUQQojMJqs9WqjxPANeXl4AjBo1isWLF6Ovr8+QIUMYPlwmkBBCCCEyE40GEA4ZMkT170aNGnH79m3c3d0pVqwYdnZyr1cIIUTmltXmGdDKdFS2trbY2tpqoyghhBAi3WWWe/3akuTGwMKFC+nTpw/6+vosXLjwi3kHDhyY4sCEEEKI9JJZ7vVrS5IbA/PmzePnn39GX1+fefPmJZpPoVBIY0AIIYTIRJLcGPj49EDcf3+cpkDTNw4KIYQQGU1WGzOg0dMEAKtXr6Zs2bLo6+ujr69P2bJlWbVqlTZjE0IIIdKFUqnU2pIZaNQYGD9+PIMGDcLR0ZEdO3awY8cOHB0dGTJkCOPHj9d2jEIIIUSWsXjxYgoVKoS+vj4ODg5cvnw50bwrV66kdu3amJqaYmpqSqNGjb6YPzEaPU2wdOlSVq5cSefOnVVprVq1ws7OjgEDBjB58mRNihVCCCEyhPR6mmDbtm04OzuzbNkyHBwcmD9/Pk2bNsXHxwcLi/gvx3J1daVz587UqFEDfX19/vrrL5o0acLNmzfJnz/p0/xr1DPw/v17KleuHC/d3t4+zV/OI4QQQmhbjBaXyMhIwsPD1ZbEXvE8d+5cfvnlF5ycnPjuu+9YtmwZhoaG/PPPPwnm37RpE3379qVChQqUKlWKVatWERMTg4uLS7Lqq1FjoGvXrixdujRe+ooVK/j55581KVIIIYT4Jk2fPp1cuXKpLdOnT4+XLyoqCnd3dxo1aqRK09HRoVGjRly4cCFJn/XmzRvev39PnjzJe+tokm8TODs7q/6tUChYtWoVR48epVq1agBcunQJPz8/unXrlqwAhBBCiIxGm/MMjB49Wu1vKICenl68fCEhIURHR2NpaamWbmlpye3bt5P0WSNHjsTa2lqtQZEUSW4MXL16Ve1ne3t7AO7fvw+AmZkZZmZm3Lx5M1kBCCGEEBmNNscM6OnpJfjHX9tmzJjB1q1bcXV1RV9fP1nbJrkxcPLkyWQHJoQQQoikMTMzI1u2bAQFBamlBwUFYWVl9cVtZ8+ezYwZMzh+/LhG7wjSeJ4BIYQQ4luVHvMM6OrqYm9vrzb47+NgwOrVqye63cyZM5kyZQqHDx9OcHB/UmjlRUVCCCHEtyS9ZiB0dname/fuVK5cmapVqzJ//nwiIiJwcnICoFu3buTPn181APGvv/5i/PjxbN68mUKFChEYGAhAzpw5yZkzZ5I/VxoDQgghRBzp9aKijh078uzZM8aPH09gYCAVKlTg8OHDqkGFfn5+6Oh86tRfunQpUVFR/Pjjj2rlTJgwgYkTJyb5c6UxIIQQQmQg/fv3p3///gmuc3V1VfvZ19dXK58pjQEhhBAijvSagTC9SGNACCGEiCOzvGBIW+RpAiGEECKLk54BIYQQIg65TSCEEEJkcen1NEF6kcZABvPi3ev0DiFDaPL2XHqHIDIQA+va6R1ChvDW/0x6hyC+UdIYEEIIIeKIyWIDCKUxIIQQQsSRtZoC8jSBEEIIkeVJz4AQQggRhzxNIIQQQmRx0hgQQgghsjiZgVAIIYQQWYr0DAghhBBxyG0CIYQQIovLajMQym0CIYQQIouTngEhhBAijqw2gFAaA0IIIUQcWW3MgNwmEEIIIbI46RkQQggh4pDbBEIIIUQWJ7cJhBBCCJGlSM+AEEIIEUdWm2cgyY2BhQsXJrnQgQMHahSMEEIIkRHEZLExAwplEkdJFC5cOGkFKhQ8ePAg2YFk182f7G2+RToKRXqHkCFktRNRiKR4638mvUPIMHKYFUnV8stYOmitrJtBl7RWVmpJcs/Aw4cPUzMOIYQQQqSTFA0gjIqKwsfHhw8fPmgrHiGEECLdxSiVWlsyA40aA2/evKFXr14YGhpSpkwZ/Pz8ABgwYAAzZszQaoBCCCFEWlNq8b/MQKPGwOjRo/Hy8sLV1RV9fX1VeqNGjdi2bZvWghNCCCFE6tPo0cLdu3ezbds2qlWrhuKzAW9lypTh/v37WgtOCCGESA+ZpXtfWzTqGXj27BkWFhbx0iMiItQaB2lt4oRhPH7kwauwexw5tJVixb7+BMTvv3Xn3p2LvA6/z/mz+6hSuYLa+t69fsbl2A5CQ27zIeopuXKZxCtj9KiBnDm1h/CX9wgJ9tZWdZLkt9+6c8fnAuFh9zh7Zh+V48QfV7u2Lbh+zZXwsHt4uB+nWbMG8fJMGD+MR77uhL28x6FDW9T2Y5061YmKfJLgYm9fXpXvx3Ytcbt8hJcv7nL3zkWcnX/TWp2TKj2OB1vbAqxYPpu7Phd4FXYPn1vnmDB+KDly5NBm1b4oNeqtp6fHwgXTCAq4wcvQO2zftgILC7MEy8qTxxTfB1cS3D+dO7fB/coxwl/e4/EjD1aumEOePKYa1zW5tL1vTE1zM3/eFG7eOM2rsHs8uHeZeXMnY2JirMqTJ48pB/ZtxM/XnYhXD3h4340F86dibJwzNaqYaq54XqffiAnUb/UzZWs2x+X0+fQOKVXJbYIkqFy5MgcOHFD9/LEBsGrVKqpXr66dyJJp+LC+9O/Xk779R1GjliMRb95wcP8m9PT0Et2mfftWzJ41gSlT51LFoRle17w5eGAT5uZ5VXkMDQ04ctSVGX8tSrQcXd0c7Px3P8uXr9dqnb6m/Y+OzJo5nqnT5uHg0Jxr1705sH+jWvyfq1bNng0bFrNm7VaqOjRj797D7NyxijLflVTlGTa0L/36OdF/wGhq1XLkTcQb9u/fqNqPFy5cwaZgRbVl9erNPHj4CHd3LwCaNq3PunWLWLFyAxUrNWTgoDEMHNCb33/vker75KP0Oh5KlSyGjo4OffuNxK5CA4YOn0ifX7oybcoordcxIalV7zmzJ9KyRWM6df6VBg3bYZ3Pip3bVyVY3soVs7l+PX6juEb1yqz9ZwFr1mzBrkJ9OnX+lSpVKrB82cyUVzwJUmPfWFtbYm1tyciRUyhfsSG9eg+hadP6rFwxR1VGTEwMe/cdpU1bJ0qXqU2v3kNo2LA2SxZnrvFVb9++o2SxIvwxtG96hyJSQZLnGfjc2bNnad68OV26dGHt2rX8+uuveHt7c/78eU6dOoW9vX2yA0npPAOPH3kwb/5y5s5bDoCJiTH+Tzzp2XsI27fvTXCb82f34XbFi0GDxwKxjRrfB24sXrKGmbMWq+WtW6c6Lsd3kte8NGFh4QmW161rB+bOmYiZxXca1yM58wycPbOPK+5eDP4s/gf33ViyZA2zZi+Ol3/TxiUYGhnSpk0PVdqZ03vxunaT/v1HA/DI1535C1Yw77P9+OTxVXr3dmb7jvj7MXv27Pg+vMKSJWv4c/oCANav+5scObLT+adPvQF9+zox1Pl3iharmqS6pbSLLiMcDx8Ndf6NX/t0o0SpGimqU1KkRr1NTIwJ9L9Gl279+e+/2C8BJUsW5eb109Ss5cilyx6qsn7t040O7R2ZOm0+x45uV9s/zkN+5dc+3ShZuqYqf7++Tgwf1o9CRSqnyv74XGofEx+1a9eS9WsXYpK7ONHR0Qnm6d+vJ0Odf6dw0SrJqkNGmWegbM3mLJg+joZ1Uv+YTkxqzzNQ1KyS1sq6H+Lx9UzpTKOegVq1auHp6cmHDx8oV64cR48excLCggsXLmjUEEipwoULki+fJS4nzqrSwsNfcfnyVao5JBxPjhw5qFTJDpcTn04upVKJy4mzVKuW9nVIrtj4y3EiTvwnTpyhWrWED2IHB3u1/ADHjp1S7aOP+/GEy6c8sfvRE4dE9oljyybkzWvKuvXbVWl6erq8exeplu/t23fY2Fhja1sgeRXVQEY7HnLlMiH0xcsUlZEUqVVv+0p26Orq4vLZceHjc59Hj56o7ZvSpYsz9o/B9Og5iJiYmHifdfGiOzY21jT//60pCwsz2rVtwaHDLimreBKk5TGRy8SY8PDXiTYE8uWzpE3r5pw+c0HD2oi0ILcJkqho0aKsXLmSy5cv4+3tzcaNGylXrlySto2MjCQ8PFxtScnrIq0sY8cvBAU9U0sPCg7Byir+2AYAM7M8ZM+eneCgELX04OBnWFmaaxxLWvkYf9w6BweHYGmZcJ2trMzj1Tco+BmW/6/vx/8HBSd9n/Rw6sTRY6d4+jRAlXb02Clat25O/fo1USgUFC9emCGD+/w/hoRj06aMdDwULVqIfn2dWLlyo8ZlJFVq1dvSypzIyMh4PSDBwc+wsorNo6ury8YNSxg5eiqPH/sn+FnnL1yha/cBbN60lLcRvvg/8SIs/BUDBv6R/MomU1odE3nzmvLHmMGsWr0p3rqNGxarxkqEv3pNn1+Ha1IVIVKFxo2B6Ohodu7cyZQpU5gyZQr//vtvkicfmj59Orly5VJblDGvkvzZnTu34WXoHdWSI4e8byk95M+fjyaN67J2zVa19NWrN7F06Vp271pHxOuHnDm9T9UNGxOj/VZyRj0erK2tOLBvIzv/3c/qfzZrvfyMVO8/p47m9u27bN78X6J5Spcuzrw5k5g6bR5VqzXn+xY/YWtbIFXunafHvjE2zsm+Peu5desOkybPibd+6LCJVHFoSuu2PShSxJbZsyakekxCc0pljNaWzECjM+TmzZu0atWKwMBASpaMHXz2119/YW5uzr59+yhbtuwXtx89ejTOzs5qaaZ5SyX58/ftO8rly1dVP+vp6QKx32wDA4NV6ZYWZnh63UywjJCQUD58+ICFpfqIaAsLcwLjfHvIiD7GbxnnG4qFhRlBQcEJbhMY+CxefS0tzFXflj7+39LCTG0/WliY43Ut/n7s3q0Dz5+/YN/+o/HWjfnjT8aOm4GVlQXPnj2nQYNaADx8+CgZtUyajHg85MtnyfFjO7hw0Z3ffh+R7O2TIq3qHRT4DD09PXLlMlHrHbCwMCcwMDZPvfo1KVe2FO3atgA+DSoOCrjO9BkLmTR5DiNHDOD8hSvMmbsMgOvXbxER8YZTrrsZP2GmWswpldbHRM6cRhzcv4lXryJo1753gl+MgoKeERT0DB+f+7wIfckp191M+3O+VusttCcmk3Tva4tGPQO9e/emTJkyPHnyBA8PDzw8PHj8+DF2dnb06dPnq9vr6elhYmKitiTnkcTXryO4f99XtXh73yEgIIgG9Wup8hgb56Rq1YpcvOSeYBnv37/Hw+Oa2jYKhYIG9Wtx8WLC22QksfFfp36c+OvXr8XFiwkPVrl0yV2tvgANG9ZW7aOHD/0ICAiifoO4+7EClxLYJ926d2Djpp2J9gjFxMTg7x/I+/fv6djhBy5cuEJISGiy6/o1Ge14sLa2wuX4Tjw8rtGr95AU3QL7krSqt7vHNaKiolQNOoASJYpia1tAladDx1+oVLkx9lWaYF+lCX1+HQZAvfptWbJ0LQCGhvrxxhJER8eoPlOb0vKYMDbOyeGDW4iKiqJ12x5ERkYmVJwaHZ3YS+/HRorIeJRKpdaWzECjngFPT0+uXLmCqemn54NNTU2ZNm0aVaokb3SstixctIoxowdy994DfH0fM2nicPz9g9iz54gqz9HD29i955Dq4jRvwUrWrJ6Hu8c13NyuMnDALxgZGbB23adZFC0tzbGysqBo0UIAlCtbilevI/Dze8qL/w8Ks7GxJk8eUwoWtCZbtmyUL18GgHv3HhIR8SbV6rxgwQpWr56Hh7sXblc8GTCgN0ZGBqxbHxv/P6vn4+8fyNhxsd2wi/5ejcvxnQwe3IdDh1zo0P4H7O3t6Nt3pKrMRYtWM3rUQO7de4jvw8dMnDgM/4Ag9uw9ovbZ9evXpEhhW9as2RIvrrx5TWnbtgWnT19AX0+Pbt070q5dSxo2+jHV9kVc6XU8WFtb4XJsJ35+Txgxcora43lx71dnlnqHh7/inzVbmT1zAi9CXxIe/ooF86dy4cIV1ZMEDx6o9/iY5c0DwK3bd1W9Cfv3H2f5spn82qcbR4+5ks/KgjlzJnH5sgcBAUGpvWtSZd98bAgYGOrTrccATEyMVXMMPHv2nJiYGJo3a4CFhRlX3L14/TqCMt+VZMaMsZw7d5lHj56ker215c2bt/g9+TQe5Kl/ELfv3CeXiTH50mAskEhdGjUGSpQoQVBQEGXKlFFLDw4OplixYloJLLlmzV6CkZEhy5bMJHduE86dc6OFYxe1VnqRIraYmeVR/bxjx17MzfIwcfwwrKzM8fK6SYuWXQj+bADdr326Mn7cUNXPrid3AdCz1xDWb4gdQT9xwnC6d+ugyuPuFttt3rDRj5w6nXojhnfs3IeZeV7Gq+L3pqVjV1X8Njb51b6JXbzoTrdu/Zk0aQRTJo/k3r2H/Ni+Nze9fVR5Zs+J3Y9LFv8Vux/Pu+EYZz8COPXozPnzbvj4JDzjZNcu7flrxjgUCgUXL7nTqHF7rlzx1P5OSER6HQ+NGtahePHCFC9eGD9f9W+cafGa7tSq99BhE4mJiWH7thXo6elx9Jgr/QeMSVZs6zdsx9jYiL59ezBr5nhevgzjpOs5Ro/5M+UVT4LU2DeVKpbDwSH26Z07t9Un4Sla3IFHj57w9u07evf6mTmzJ6Knp8vjJwHs3n2Qv2Ym/GhiRnXj9l16Dvj0xWHmohUA/NC8EdPGDk1ss0wrq90mSPI8A+Hhn+4Vnj17lhEjRjBx4kSqVasGwMWLF5k8eTIzZszg+++/T3YgaXGhzAySM8/AtyyrTQUqRFJklHkGMoLUnmcgv2mZr2dKoqcvEh6XkpEkuTGgo6Ojdl/v42Yf0z7/ObHna79EGgOxpDEQSxoDQsQnjYFPpDGgXUm+TXDy5MnUjEMIIYTIMLLaF5IkNwbq1q2bmnEIIYQQGUZmmTlQW1I0E8ebN2/w8/MjKipKLd3Ozi5FQQkhhBAi7WjUGHj27BlOTk4cOnQowfWajBkQQgghMorMMj+Atmg06dDgwYN5+fIlly5dwsDAgMOHD7Nu3TqKFy/O3r0Jv/1LCCGEyCxiUGptyQw06hk4ceIEe/bsoXLlyujo6GBra0vjxo0xMTFh+vTptGjRQttxCiGEECKVaNQzEBERgYVF7IxTpqamPHsWO7NauXLl8PDI+O9tFkIIIb4kq01HrFFjoGTJkvj4xM5aV758eZYvX87Tp09ZtmwZ+fLl02qAQgghRFqLUSq1tmQGGt0mGDRoEAEBse+vnzBhAs2aNWPjxo3o6uqybt06rQYohBBCpLXM8o1eW5I8A+GXvHnzhtu3b1OwYEHMzMy+vkECZAbCWDIDYazM0poWIi3JDISfpPYMhKY5tfeenRev72mtrNSS5J4BZ2fnJBc6d+5cjYIRQgghMoLM8hSAtiS5MXD16tUk5dP2e8mFEEKItJbVbhPIuwmEEEKILC5F0xELIYQQ36KsNm5JGgNCCCFEHFntRUUazTMghBBCiG+H9AwIIYQQcchtAiGEECKLy2pPE8htAiGEECKLk54BIYQQIo6sNoBQGgNCCCFEHHKbQAghhMji0vMVxosXL6ZQoULo6+vj4ODA5cuXv5h/x44dlCpVCn19fcqVK8fBgweT/ZnSGBBCCCEyiG3btuHs7MyECRPw8PCgfPnyNG3alODg4ATznz9/ns6dO9OrVy+uXr1K69atad26NTdu3EjW52rlrYXaIG8tjCVvLYyV1R7rESIp5K2Fn6T2Wwu1+Tcp4tUDIiMj1dL09PTQ09OLl9fBwYEqVarw999/AxATE4ONjQ0DBgxg1KhR8fJ37NiRiIgI9u/fr0qrVq0aFSpUYNmyZUkPUimUSqVS+e7dO+WECROU7969S+9Q0pXsh1iyH2LJfogl+yGW7AfNTJgwQQmoLRMmTIiXLzIyUpktWzblrl271NK7deumbNWqVYJl29jYKOfNm6eWNn78eKWdnV2yYswwPQPpLTw8nFy5chEWFoaJiUl6h5NuZD/Ekv0QS/ZDLNkPsWQ/aCYyMjJJPQP+/v7kz5+f8+fPU716dVX6iBEjOHXqFJcuXYpXtq6uLuvWraNz586qtCVLljBp0iSCgoKSHKM8TSCEEEKkosRuCWQkMoBQCCGEyADMzMzIli1bvG/0QUFBWFlZJbiNlZVVsvInRhoDQgghRAagq6uLvb09Li4uqrSYmBhcXFzUbht8rnr16mr5AY4dO5Zo/sTIbYL/09PTY8KECRm+Kye1yX6IJfshluyHWLIfYsl+SH3Ozs50796dypUrU7VqVebPn09ERAROTk4AdOvWjfz58zN9+nQABg0aRN26dZkzZw4tWrRg69atXLlyhRUrViTrc2UAoRBCCJGB/P3338yaNYvAwEAqVKjAwoULcXBwAKBevXoUKlSItWvXqvLv2LGDsWPH4uvrS/HixZk5cybff/99sj5TGgNCCCFEFidjBoQQQogsThoDQgghRBYnjQEhhBAii5PGQAIKFSrE/Pnz0zuMVPOt1y8lFAoFu3fvTtcY6tWrx+DBgzXatkePHrRu3Vqr8WR2X9ufWfF8SM4x5urqikKh4OXLlyn6zKy4nzMTebQwAW5ubhgZGaV3GKlGm/VzdXWlfv36vHjxgty5c2ulTCHi6tGjBy9fvkyVhtq3fr4LkRTSGEiAubl5eoeQqr71+onU8/79e3LkyJHeYWiVnA9CfMO3CRLqkqpQoQITJ05EqVQyceJEChYsiJ6eHtbW1gwcODDRbefOnUu5cuUwMjLCxsaGvn378vr16zSqSfLVq1eP/v37079/f3LlyoWZmRnjxo3j41Okn9fva/tiw4YNVK5cGWNjY6ysrPjpp59U79X29fWlfv36AJiamqJQKOjRo0ea1vVL6tWrx8CBAxkxYgR58uTBysqKiRMnqtbfvXuXOnXqoK+vz3fffcexY8filTFy5EhKlCiBoaEhRYoUYdy4cbx//x6Irb+Ojg5XrlxR22b+/PnY2toSExOjlXocOHCAXLlysWnTJh4/fkyHDh3InTs3efLk4YcffsDX1zfRbQ8fPkytWrXInTs3efPmpWXLlty/f1+1Pioqiv79+5MvXz709fWxtbVVTWYCsbdNli5dSqtWrTAyMmLatGlER0fTq1cvChcujIGBASVLlmTBggWqbW7cuIGOjg7Pnj0DIDQ0FB0dHTp16qTKM3XqVGrVqgXw1fImTpzIunXr2LNnDwqFAoVCgaura7L24YcPH5J0PsCXz/eIiAhMTEzYuXOnWvm7d+/GyMiIV69eAV8+bjKaL53jnzt37hx2dnbo6+tTrVo1bty4obb+7Nmz1K5dGwMDA2xsbBg4cCARERFpVQ2RQt9sY+BL/v33X+bNm8fy5cu5e/cuu3fvply5conm19HRYeHChdy8eZN169Zx4sQJRowYkYYRJ9+6devInj07ly9fZsGCBcydO5dVq1bFy/e1ffH+/XumTJmCl5cXu3fvxtfXV/UH38bGhn///RcAHx8fAgIC1C7iGcG6deswMjLi0qVLzJw5k8mTJ3Ps2DFiYmJo27Yturq6XLp0iWXLljFy5Mh42xsbG7N27Vq8vb1ZsGABK1euZN68eUDsH5FGjRqxZs0atW3WrFlDjx490NFJ+em1efNmOnfuzKZNm+jQoQNNmzbF2NiYM2fOcO7cOXLmzEmzZs2IiopKcPuIiAicnZ25cuUKLi4u6Ojo0KZNG1VDZeHChezdu5ft27fj4+PDpk2bKFSokFoZEydOpE2bNly/fp2ePXsSExNDgQIF2LFjB97e3owfP54xY8awfft2AMqUKUPevHk5deoUAGfOnFH7GeDUqVPUq1cP4KvlDRs2jA4dOtCsWTMCAgIICAigRo0aydqPST0f4Mvnu5GREZ06dUrwd/7jjz9ibGwMfPm4yWi+dI5/bvjw4cyZMwc3NzfMzc1xdHRUNXDu379Ps2bNaNeuHdeuXWPbtm2cPXuW/v37p3FthMaS9cLjTMTW1jbeO57Lly+vnDBhgnLOnDnKEiVKKKOiopK87ed27NihzJs3rxaj1a66desqS5curYyJiVGljRw5Ulm6dGmlUqlev6/ti7jc3NyUgPLVq1dKpVKpPHnypBJQvnjxQqt10Ia6desqa9WqpZZWpUoV5ciRI5VHjhxRZs+eXfn06VPVukOHDimBeO8S/9ysWbOU9vb2qp+3bdumNDU1Vb3f3d3dXalQKJQPHz5MUdyDBg1S/v3338pcuXIpXV1dlUqlUrlhwwZlyZIl1X6vkZGRSgMDA+WRI0eUSqVS2b17d+UPP/yQaNnPnj1TAsrr168rlUqlcsCAAcoGDRqolfk5QDl48OCvxtyvXz9lu3btVD+3bdtW2a9fP6VSqVQOHjxYOXz4cKWpqany1q1byqioKKWhoaHy6NGjSS7va/X6kuScDwmJe75funRJmS1bNqW/v79SqVQqg4KClNmzZ1f9nhIS97hJbx+PsYQkdo5v3bpVlef58+dKAwMD5bZt25RKpVLZq1cvZZ8+fdTKOXPmjFJHR0f59u1bpVL59f0s0leW7Blo3749b9++pUiRIvzyyy/s2rWLDx8+JJr/+PHjNGzYkPz582NsbEzXrl15/vw5b968ScOok6datWooFArVz9WrV+fu3btER0er5fvavnB3d8fR0ZGCBQtibGxM3bp1AfDz80ubiqSQnZ2d2s/58uUjODiYW7duYWNjg7W1tWpdQi/22LZtGzVr1sTKyoqcOXMyduxYtbq3bt2abNmysWvXLgDWrl1L/fr14327Tq6dO3cyZMgQjh07ptrnXl5e3Lt3D2NjY3LmzEnOnDnJkycP7969U+v6/9zdu3fp3LkzRYoUwcTERBXXxzr06NEDT09PSpYsycCBAzl69Gi8MipXrhwvbfHixdjb22Nubk7OnDlZsWKF2n6pW7euqiv/1KlTNGjQgDp16uDq6oqbmxvv37+nZs2aSS4vpZJ6PsDXz/eqVatSpkwZ1q1bB8DGjRuxtbWlTp06qjK+dtxkJEk9xz8/P/LkyUPJkiW5desWEHtsrl27VnVc5syZk6ZNmxITE8PDhw/TrjJCY99sY0BHR0d1T/Cjj11aNjY2+Pj4sGTJEgwMDOjbty916tRJ8J6er68vLVu2xM7Ojn///Rd3d3cWL14MkGjXbGbypX0RERFB06ZNMTExYdOmTbi5uan+6GWWuscd7KZQKJJ8L//ChQv8/PPPfP/99+zfv5+rV6/yxx9/qNVdV1eXbt26sWbNGqKioti8eTM9e/ZMcdwVK1bE3Nycf/75R3Ucv379Gnt7ezw9PdWWO3fu8NNPPyVYjqOjI6GhoaxcuZJLly5x6dIl4NPvr1KlSjx8+JApU6bw9u1bOnTowI8//qhWRtyR9lu3bmXYsGH06tWLo0eP4unpiZOTk9p+qVevHt7e3ty9exdvb29q1apFvXr1cHV15dSpU1SuXBlDQ8Mkl5dWknq+9+7dWzU3/Jo1a3ByclI1NpJy3GQU2jrHX79+za+//qp2XHp5eXH37l2KFi2aWuELLfpmnyYwNzcnICBA9XN4eLhaC9XAwABHR0ccHR3p168fpUqV4vr161SqVEmtHHd3d2JiYpgzZ47qHvDHe5kZ2ceL/kcXL16kePHiZMuWLV7exPaFUqnk+fPnzJgxAxsbG4B4g+V0dXUBEvyGlZGVLl2ax48fExAQQL58+YDYffS58+fPY2tryx9//KFKe/ToUbyyevfuTdmyZVmyZAkfPnygbdu2KY6vaNGizJkzh3r16pEtWzb+/vtvKlWqxLZt27CwsMDExOSrZTx//hwfHx9WrlxJ7dq1gdhBXnGZmJjQsWNHOnbsyI8//kizZs0IDQ0lT548CZZ77tw5atSoQd++fVVpcXsmypUrh6mpKVOnTqVChQrkzJmTevXq8ddff/HixQvVeIGklqerq5uiYyyp50NSz/cuXbowYsQIFi5ciLe3N927d1etS+pxkxHcvn37q+f4RxcvXqRgwYIAvHjxgjt37lC6dGkgtlHp7e1NsWLF0iZwoXXfbM9AgwYN2LBhA2fOnOH69et0795ddeKvXbuW1atXc+PGDR48eMDGjRsxMDDA1tY2XjnFihXj/fv3LFq0iAcPHrBhwwaWLVuW1tVJNj8/P5ydnfHx8WHLli0sWrSIQYMGxcv3pX1RsGBBdHV1VXXfu3cvU6ZMUdve1tYWhULB/v37efbsWYZ+yuJzjRo1okSJEnTv3h0vLy/OnDmjdvEGKF68OH5+fmzdupX79++zcOFC1bemz5UuXZpq1aoxcuRIOnfujIGBgVZiLFGiBCdPnuTff/9l8ODB/Pzzz5iZmfHDDz9w5swZHj58iKurKwMHDuTJkyfxtjc1NSVv3rysWLGCe/fuceLECZydndXyzJ07ly1btnD79m3u3LnDjh07sLKy+uKcEcWLF+fKlSscOXKEO3fuMG7cONzc3NTyKBQK6tSpw6ZNm1R/+O3s7IiMjMTFxUXVFZ3U8goVKsS1a9fw8fEhJCQk2SPzk3o+JPV8NzU1pW3btgwfPpwmTZpQoEABtfok5bjJCJJyjn80efJkXFxcuHHjBj169MDMzEw1wdXIkSM5f/48/fv3x9PTk7t377Jnzx4ZQJiZpPOYhVQTFham7Nixo9LExERpY2OjXLt2rWoA4a5du5QODg5KExMTpZGRkbJatWrK48ePq7aNO9Bl7ty5ynz58ikNDAyUTZs2Va5fvz7DDppTKmMHB/Xt21f522+/KU1MTJSmpqbKMWPGqAZQfV6/r+2LzZs3KwsVKqTU09NTVq9eXbl3714loLx69aoqz+TJk5VWVlZKhUKh7N69exrW9MsSGiT1ww8/qGL08fFR1qpVS6mrq6ssUaKE8vDhw/EGEA4fPlyZN29eZc6cOZUdO3ZUzps3T5krV654n7V69WoloLx8+bLW4/b29lZaWFgonZ2dlQEBAcpu3bopzczMlHp6esoiRYoof/nlF2VYWJhSqYw/0O7YsWPK0qVLK/X09JR2dnZKV1dXtTquWLFCWaFCBaWRkZHSxMRE2bBhQ6WHh4dq+7j7Q6lUKt+9e6fs0aOHMleuXMrcuXMrf//9d+WoUaOU5cuXV8s3b948JaA8dOiQKu2HH35QZs+eXTU4LanlBQcHKxs3bqzMmTOnElCePHkyWfszqeeDUpn0893FxUUJKLdv3x7vM5N63KSXz4+xr53jHwcQ7tu3T1mmTBmlrq6usmrVqkovLy+1Mi9fvqz6HRkZGSnt7OyU06ZNU62XAYQZm7zC+BtUr149KlSoIFN/pqEpU6awY8cOrl27lt6hiDSyYcMGhgwZgr+/v+p2mRCZ1Tc7ZkCItPD69Wt8fX35+++/mTp1anqHI9LAmzdvCAgIYMaMGfz666/SEBDfhG92zIAQaaF///7Y29tTr149rTxFIDK+mTNnUqpUKaysrBg9evT/2rNjGgAAGIZh/FmPRKUdsRHsjLrvc2DCmwAA4iwDABAnBgAgTgwAQJwYAIA4MQAAcWIAAOLEAADEiQEAiDsPHXeTeRDAoQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# check correlation between numerical data and the label\n",
    "import seaborn as sns\n",
    "sns.heatmap(numeric_col.corr(), annot=True, vmin=0, vmax=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>296</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>229</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>114</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>97</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>11</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>735</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          dati2  typefaskes  jenkel  jenispel  jenispulang  cbg\n",
       "0            15          16       1         1            0  780\n",
       "1            15           0       0         0            0  102\n",
       "2            15          16       1         1            0  782\n",
       "3            15          16       1         1            0  782\n",
       "4            15           0       1         1            0  782\n",
       "...         ...         ...     ...       ...          ...  ...\n",
       "11291759    296           1       1         0            0  738\n",
       "11291760    229           0       1         0            0  727\n",
       "11291761    114           1       0         0            0  717\n",
       "11291762     97          24       0         0            0  717\n",
       "11291763     11          25       1         0            0  735\n",
       "\n",
       "[11291764 rows x 6 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# labelencode all categorical data and save to variable\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "categorical = pd.DataFrame()\n",
    "categorical[\"dati2\"] = le.fit_transform(samp[\"dati2\"])\n",
    "categorical[\"typefaskes\"] = le.fit_transform(samp[\"typefaskes\"])\n",
    "categorical[\"jenkel\"] = le.fit_transform(samp[\"jenkel\"])\n",
    "categorical[\"jenispel\"] = le.fit_transform(samp[\"jenispel\"])\n",
    "categorical[\"jenispulang\"] = le.fit_transform(samp[\"jenispulang\"])\n",
    "categorical[\"cbg\"] = le.fit_transform(samp[\"cbg\"])\n",
    "categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dati2          0.000000e+00\n",
       "typefaskes     0.000000e+00\n",
       "cbg            0.000000e+00\n",
       "jenispulang    5.948716e-37\n",
       "jenkel         9.271066e-17\n",
       "jenispel       5.667109e-09\n",
       "dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the p value each column to label using chi square\n",
    "from sklearn.feature_selection import chi2\n",
    "res_chi2 = chi2(categorical, samp[\"label\"])\n",
    "p_val = pd.Series(res_chi2[1], index=categorical.columns)\n",
    "p_val.sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    11136086\n",
       "1      155678\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# imbalance data\n",
    "samp[\"label\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Proc and Diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ewwwr\\AppData\\Local\\Temp\\ipykernel_11412\\3760645589.py:5: DtypeWarning: Columns (1) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  proc = pd.read_csv(\"temp_data/1/train_proc_cleaned.csv\")\n"
     ]
    }
   ],
   "source": [
    "# load all dataset for samp only using the id and label for merging proc and diag\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "samp = pd.read_csv(\"temp_data/1/train_samp_cleaned.csv\", usecols=[\"id\", \"label\"])\n",
    "proc = pd.read_csv(\"temp_data/1/train_proc_cleaned.csv\")\n",
    "diag = pd.read_csv(\"temp_data/1/train_diag_cleaned.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>90.59</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>69.01</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>88.78</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>90.59</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27</td>\n",
       "      <td>88.78</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120158</th>\n",
       "      <td>11426915</td>\n",
       "      <td>93.57</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120159</th>\n",
       "      <td>11426927</td>\n",
       "      <td>89.52</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120160</th>\n",
       "      <td>11426927</td>\n",
       "      <td>96.55</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120161</th>\n",
       "      <td>11426927</td>\n",
       "      <td>87.39</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120162</th>\n",
       "      <td>11426927</td>\n",
       "      <td>93.96</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12120163 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id   proc  nproc\n",
       "0                6  90.59    3.0\n",
       "1                6  69.01    3.0\n",
       "2                6  88.78    3.0\n",
       "3               27  90.59    2.0\n",
       "4               27  88.78    2.0\n",
       "...            ...    ...    ...\n",
       "12120158  11426915  93.57    1.0\n",
       "12120159  11426927  89.52    4.0\n",
       "12120160  11426927  96.55    4.0\n",
       "12120161  11426927  87.39    4.0\n",
       "12120162  11426927  93.96    4.0\n",
       "\n",
       "[12120163 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# counting amount of proc on each id\n",
    "# if the calculation is null its actualy 1 caused by .duplicated method only counting\n",
    "# the duplicate value so i fill null value with 1\n",
    "\n",
    "temp = dict(proc[proc[\"id\"].duplicated(keep=False)][\"id\"].value_counts())\n",
    "temp = pd.DataFrame({\n",
    "    \"id\":list(temp.keys()),\n",
    "    \"nproc\":list(temp.values())\n",
    "})\n",
    "proc = pd.merge(proc, temp, on=\"id\", how=\"left\")\n",
    "proc[\"nproc\"].fillna(1, inplace=True)  \n",
    "del temp\n",
    "gc.collect()\n",
    "proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>90.59</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>90.59</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>29</td>\n",
       "      <td>88.76</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>30</td>\n",
       "      <td>90.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>33</td>\n",
       "      <td>90.59</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120155</th>\n",
       "      <td>11426788</td>\n",
       "      <td>88.78</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120156</th>\n",
       "      <td>11426822</td>\n",
       "      <td>23.3</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120157</th>\n",
       "      <td>11426839</td>\n",
       "      <td>4.81</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120158</th>\n",
       "      <td>11426915</td>\n",
       "      <td>93.57</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120159</th>\n",
       "      <td>11426927</td>\n",
       "      <td>89.52</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6372051 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id   proc  nproc\n",
       "0                6  90.59    3.0\n",
       "3               27  90.59    2.0\n",
       "5               29  88.76    1.0\n",
       "6               30  90.59    1.0\n",
       "7               33  90.59    2.0\n",
       "...            ...    ...    ...\n",
       "12120155  11426788  88.78    1.0\n",
       "12120156  11426822   23.3    1.0\n",
       "12120157  11426839   4.81    1.0\n",
       "12120158  11426915  93.57    1.0\n",
       "12120159  11426927  89.52    4.0\n",
       "\n",
       "[6372051 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop the duplicate id because we have got the amount of proc in nproc column\n",
    "proc.drop_duplicates(subset=\"id\", inplace=True)\n",
    "proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>9983563</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>11053870</td>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>7461049</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>1075162</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>102794</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  label  nproc\n",
       "0           165666      0    0.0\n",
       "1          1010828      0    1.0\n",
       "2           166042      0    0.0\n",
       "3           168937      0    0.0\n",
       "4          1005899      0    0.0\n",
       "...            ...    ...    ...\n",
       "11291759   9983563      0    4.0\n",
       "11291760  11053870      0    5.0\n",
       "11291761   7461049      0    1.0\n",
       "11291762   1075162      0    1.0\n",
       "11291763    102794      0    0.0\n",
       "\n",
       "[11291764 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# left join the nproc column with the samp\n",
    "# fillna with 0 because if there is null it means the patient doesnt have any procedure\n",
    "\n",
    "proc.drop(\"proc\", axis=1, inplace=True)\n",
    "df = pd.merge(samp, proc, on=\"id\", how=\"left\")\n",
    "df[\"nproc\"].fillna(0, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>diag</th>\n",
       "      <th>levelid</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>111</td>\n",
       "      <td>O86.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>133</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>231</td>\n",
       "      <td>E11.9</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>231</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>248</td>\n",
       "      <td>O82.1</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905250</th>\n",
       "      <td>11426932</td>\n",
       "      <td>M79.1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905251</th>\n",
       "      <td>11426946</td>\n",
       "      <td>N30.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905252</th>\n",
       "      <td>11426947</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905253</th>\n",
       "      <td>11426947</td>\n",
       "      <td>E78.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905254</th>\n",
       "      <td>11426952</td>\n",
       "      <td>M54.5</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5905255 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id   diag  levelid  nsdiag\n",
       "0             111  O86.0        2     1.0\n",
       "1             133    K30        2     1.0\n",
       "2             231  E11.9        2     2.0\n",
       "3             231    K30        2     2.0\n",
       "4             248  O82.1        2     2.0\n",
       "...           ...    ...      ...     ...\n",
       "5905250  11426932  M79.1        2     1.0\n",
       "5905251  11426946  N30.0        2     1.0\n",
       "5905252  11426947    K30        2     2.0\n",
       "5905253  11426947  E78.0        2     2.0\n",
       "5905254  11426952  M54.5        2     1.0\n",
       "\n",
       "[5905255 rows x 4 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# same as before counting the secondary diagnose\n",
    "# and fill the null with 1\n",
    "temp = dict(diag[diag[\"id\"].duplicated(keep=False)][\"id\"].value_counts())\n",
    "temp = pd.DataFrame({\n",
    "    \"id\":list(temp.keys()),\n",
    "    \"nsdiag\":list(temp.values())\n",
    "})\n",
    "diag = pd.merge(diag, temp, on=\"id\", how=\"left\")\n",
    "diag[\"nsdiag\"].fillna(1, inplace=True)\n",
    "del temp\n",
    "gc.collect()\n",
    "diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>diag</th>\n",
       "      <th>levelid</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>111</td>\n",
       "      <td>O86.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>133</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>231</td>\n",
       "      <td>E11.9</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>248</td>\n",
       "      <td>O82.1</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>259</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905249</th>\n",
       "      <td>11426929</td>\n",
       "      <td>M54.5</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905250</th>\n",
       "      <td>11426932</td>\n",
       "      <td>M79.1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905251</th>\n",
       "      <td>11426946</td>\n",
       "      <td>N30.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905252</th>\n",
       "      <td>11426947</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905254</th>\n",
       "      <td>11426952</td>\n",
       "      <td>M54.5</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3815183 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id   diag  levelid  nsdiag\n",
       "0             111  O86.0        2     1.0\n",
       "1             133    K30        2     1.0\n",
       "2             231  E11.9        2     2.0\n",
       "4             248  O82.1        2     2.0\n",
       "6             259    K30        2     1.0\n",
       "...           ...    ...      ...     ...\n",
       "5905249  11426929  M54.5        2     1.0\n",
       "5905250  11426932  M79.1        2     1.0\n",
       "5905251  11426946  N30.0        2     1.0\n",
       "5905252  11426947    K30        2     2.0\n",
       "5905254  11426952  M54.5        2     1.0\n",
       "\n",
       "[3815183 rows x 4 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dropping the duplicate id because we have got the nsdiag\n",
    "diag.drop_duplicates(subset=\"id\", inplace=True)\n",
    "diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>9983563</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>11053870</td>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>7461049</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>1075162</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>102794</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  label  nproc  nsdiag\n",
       "0           165666      0    0.0     0.0\n",
       "1          1010828      0    1.0     1.0\n",
       "2           166042      0    0.0     2.0\n",
       "3           168937      0    0.0     2.0\n",
       "4          1005899      0    0.0     0.0\n",
       "...            ...    ...    ...     ...\n",
       "11291759   9983563      0    4.0     1.0\n",
       "11291760  11053870      0    5.0     1.0\n",
       "11291761   7461049      0    1.0     1.0\n",
       "11291762   1075162      0    1.0     0.0\n",
       "11291763    102794      0    0.0     0.0\n",
       "\n",
       "[11291764 rows x 4 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# left join the nsdiag with other\n",
    "# fillna with 0 because if there is null it means the patient doesnt have any secondary diagnose\n",
    "diag.drop([\"diag\", \"levelid\"], axis=1, inplace=True)\n",
    "df = pd.merge(df, diag, on=\"id\", how=\"left\")\n",
    "df[\"nsdiag\"].fillna(0, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop the id column\n",
    "df.drop(\"id\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          label  nproc  nsdiag\n",
       "0             0    0.0     0.0\n",
       "1             0    1.0     1.0\n",
       "2             0    0.0     2.0\n",
       "3             0    0.0     2.0\n",
       "4             0    0.0     0.0\n",
       "...         ...    ...     ...\n",
       "11291759      0    4.0     1.0\n",
       "11291760      0    5.0     1.0\n",
       "11291761      0    1.0     1.0\n",
       "11291762      0    1.0     0.0\n",
       "11291763      0    0.0     0.0\n",
       "\n",
       "[11291764 rows x 3 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# view data\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot: >"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGiCAYAAAB6c8WBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABEP0lEQVR4nO3deVxN6R8H8M9tu4WytcmSPWNJFFnHMJlmUGNWsmfPFjHIVlkma7ZBYxsGM7IMYxvDNJpBKUrFlBBZWwdFUeqe3x/95nJvoa5b9+Z83vM6r5f73Oc85znNpe/9PsuRCIIggIiIiERLR9MdICIiIs1iMEBERCRyDAaIiIhEjsEAERGRyDEYICIiEjkGA0RERCLHYICIiEjkGAwQERGJHIMBIiIikWMwQEREJHIMBoiIiLTE33//DRcXF1hZWUEikeDgwYNvPCckJARt27aFVCpF48aNsW3btlJfl8EAERGRlsjOzkbr1q2xbt26EtW/efMmevfuje7duyM6OhqTJ0/GyJEj8fvvv5fquhI+qIiIiEj7SCQSHDhwAH379n1lnRkzZuDo0aO4fPmyvKx///549OgRjh8/XuJrMTNARERUhnJzc5GVlaVw5ObmqqXtsLAwODk5KZQ5OzsjLCysVO3oqaU3avA844amu0BaxMiqq6a7QERaLD/vXpm2r87fSf7f/Qg/Pz+FMh8fH/j6+r512ykpKbCwsFAos7CwQFZWFp4+fQojI6MStaM1wQAREZHWkBWorSlvb294eXkplEmlUrW1rw4MBoiIiMqQVCots1/+lpaWSE1NVShLTU2FiYlJibMCAIMBIiKiogSZpntQIh07dsSxY8cUyk6ePImOHTuWqh1OICQiIlImk6nvKIUnT54gOjoa0dHRAAqXDkZHR+P27dsACocchgwZIq8/duxY3LhxA9OnT8eVK1ewfv167NmzB1OmTCnVdZkZICIiUiJoKDNw4cIFdO/eXf76v7kGQ4cOxbZt25CcnCwPDACgQYMGOHr0KKZMmYLVq1ejTp062Lx5M5ydnUt1Xa3ZZ4CrCehlXE1ARK9T1qsJ8u7/o7a2DKxaqK2tssLMABERkbJSpvcrOgYDREREyirIBEJ14QRCIiIikWNmgIiISJkaNx2qCBgMEBERKeMwAREREYkJMwNERETKuJqAiIhI3DS16ZCmcJiAiIhI5JgZICIiUsZhAiIiIpET2TABgwEiIiJlIttngHMGiIiIRI6ZASIiImUcJiAiIhI5kU0g5DABERGRyDEzQEREpIzDBERERCLHYQIiIiISE2YGiIiIlAiCuPYZYDBARESkTGRzBjhMQEREJHLMDBARESkT2QRCBgNERETKRDZMwGCAiIhIGR9URERERGLCzAAREZEyDhMQERGJnMgmEHKYgIiISOSYGSAiIlLGYQIiIiKR4zABERERiQkzA0RERMpElhlgMEBERKREbE8t5DABERGRyDEzQEREpIzDBERERCLHpYVEREQiJ7LMAOcMEBERiRwzA0RERMo4TEBERCRyHCYgIiIiMSlxZmDNmjUlbnTSpEkqdYaIiEgrcJigeCtXrixRPYlEwmCAiIgqNpENE5Q4GLh582ZZ9oOIiIg05K3mDOTl5SEhIQH5+fnq6g8REZHmyWTqOyoAlYKBnJwcjBgxApUqVUKLFi1w+/ZtAMDEiROxePFitXaQiIio3Aky9R0VgErBgLe3N2JiYhASEgJDQ0N5uZOTE4KCgtTWOSIiIip7Ku0zcPDgQQQFBaFDhw6QSCTy8hYtWiAxMVFtnSMiItKICpLeVxeVMgPp6ekwNzcvUp6dna0QHFDJXIi+hPHTfdDddSBadv4EwX+HarpLpCJfn2m4cysKjzOv4/ffdqNx4wZvPMdj7FBcv3oOT7ISEXrmMNo52Cm8L5VKsWb1IqQmX8ajB1exJ2gjzM1NFeo42LfGieNByEiLQ3rqPzh2ZBdsbZsr1PmoZzecPX0YD/9NQPK9WOwJ2ghr6zpvfc/0apr6PADAkMFfIyryJJ5kJeL+3RisWb1I/p61dR3k590rcji2b/vW9/zO4DDBmzk4OODo0aPy1/8FAJs3b0bHjh3V0zMRefr0GWwaN8TsqeM03RV6C99MG4cJ44dj3ISZ6NTFBdk5OTh2ZBekUukrz/nqK1csX+aDBQsD0M7xY8TExuHY0V0wM6spr7NiuS/69O6J/m5j0OPDL2BVyxL79myWv1+5ciUcPbILt+/cQ6cuLujW/TM8fpKNY0d2QU+vMPlXv35d/LJ/K06FnIV9u4/Qq/cAmNasgb0vtUPqpanPAwBM9hyNBfOnY+mydbC16wHnT/rjxMmQItf7yLkfate1kx+RUbFqu/8KT2QTCCWCIAilPenMmTP45JNPMGjQIGzbtg1jxoxBXFwcQkND8ddff8He3r7UHXmecaPU57yLWnb+BKv95+LD9ztpuisaZWTVVdNdKLU7t6KwctX3CFj5PQDAxMQY9+9GY/jIKdiz51Cx54SeOYzzF2LgOXkOgMLAOunGeaxb/wOWLlsHExNjpNyPxaAhE/DLL4UBuI1NI/xz6W907uKC8Igo2Le1Rfi531C/YTvcvXsfANCyZTNERwXD5r3OSExMwuef98auHetQqUoD/PdXvk/vnvhl/1ZUqtKAK4LKgKY+D9WqVcXtpEj0/WwY/jx1ptjrWFvXQeK1cNi3+wgxMf+Uwd2Xvfy8e2Xa/tMD6psMb/TZTLW1VVZUygx06dIF0dHRyM/PR6tWrXDixAmYm5sjLCxMpUCAqKJr0KAeatWyQPCfL/7xzcp6jIiIi+jgWPzfCX19fbRta4vgP0/LywRBQPCfZ9ChQ+E59m1tYWBggODgF3USEhJx69ZdeZ2Eq4nIyHiA4e79oa+vD0NDQ7gPc0Nc/FUkJd0BAERFxUImk2HY0H7Q0dGBiYkxBg78AsHBpxkIlAFNfh6cnN6Hjo4EVrUtcSk2BEk3LuDnnwJRp45VkWse2P8D7t+NwV+nDqBPn55qufd3hsiGCVR+UFGjRo2wadMmlc7Nzc1Fbm6uQplObu5r02dE2szSonAOTWpqukJ5aloGLC2Lzq8BAFPTGtDT00NaaoZCeVpaOprZNAIAWFiaITc3F5mZWUXqWFqaAQCePMnGhz2/xP69WzB71mQAwLXrN9Gr9wAUFBQAAJKS7uCTXgPw80+B2LB+CfT09BAWdgF9XAe/3Y1TsTT5eWjYoB50dHQwc8ZEeHnNQ2bmY8z3m47jv/2MNm2d8Pz5czx5ko1p3/ghNPQ8ZDIZPv+8F37ZtxWffzkcR46cVMvPoMKrIOl9dVF506GCggLs27cPCxYswIIFC7B///4Sf8Pw9/dH1apVFY4lqwNV7QpRuXNz+wyPHlyVH/r6mnsAqKGhITZ9vxyhYRfQuYsL3u/WF//8k4BDv/4oX/prYWGGwMBl2LFzLzp06oXuPT5HXl4e9uzeqLF+v0u06fOgo6MDAwMDTJkyFydO/oXwiCgMHDwOTRo3QPcPCocf//33IVat3oiI8xdxITIGs2b7Y9dPv2Cal4fG+k2apdIn9p9//oGrqytSUlJgY2MDAFiyZAnMzMxw+PBhtGzZ8rXne3t7w8vLS6FM53HZjv8QqdPhwycQEXFR/loqNQBQ+Es3JSVNXm5hboroV4zJZmQ8QH5+PswtFGeCm5ubIeX/3yhTU9IhlUpRtaqJwrdBc3MzpKQU1nHr3xfW1nXRuaurfD7AoMHjkZEWB1fXj7BnzyGM8xiGzMwszPR+MaN8yLBJuHXzAhzbt0V4RNTb/DhET5s+D8nJqQCAuPhrCm1nZDxA3bq1X3kPERFRcPqw4s3VKTPMDLzZyJEj0aJFC9y9exdRUVGIiorCnTt3YGtri9GjR7/xfKlUChMTE4WDQwRUkTx5ko3ExCT5ERd3FcnJqejRvYu8jrFxFbRv3wbnwiOLbeP58+eIiopVOEcikaBH9y44d67wnMioWOTl5aFHjxd1mjZtBGvrOvI6lSoZQSaT4eW5wP+91tEp/CteycgIgtI/bv8NIfxXh1SnTZ+H0LALAACbpo3kdapXrwZT0xq4ffvuK++hdesWSH4pcBE9QVDfUQGolBmIjo7GhQsXUL16dXlZ9erVsWjRIrRr105tnROLnJynuP3/WeAAcO9+Kq5cTURVE2PUesX4ImmfNWs3Y5b3JFy7fgNJSXfg5/sN7t9Pxa+//i6vc+J4EA7++hvWb9gGAFi5ehN+2LISkVGxOH/+IiZNHIXKlY2wbXvhTp5ZWY+x9YfdWL7UBw8fPEJW1mOsXrUQYWEX5N/m/wj+G0sWz8HaNd9i3fqt0NHRwfRvJiA/Px8hIYV7Vhz7LRienqMwZ/Zk7A76FcZVKmPhgplISrqDi9GXy/cHJRKa+jxcu3YDvx46joAAP3h4zEDW48dYtNAbVxKu49T/Pw+DB3+FvLw8RP////1nfXvBfVh/jB4zrRx/QqRNVAoGmjZtitTUVLRo0UKhPC0tDY0bN1ZLx8Tk8pVrGD5xhvz10rWF47iffuKERXOmaqpbVErLlq9H5cqVELh+KapVM8HZs+fR22WQwmTZhg2tYWpaQ/56795DMDOtAd9502BpaYaYmH/Qu88gpKW9mEQ2dZovZDIZ9gRthFQqxYmTIZgwcZb8/YSERPT9bBjmzvHCmb8PQSaTITq6sJ3/UtSnQs5i0JDxmDZ1HKZNHYecnKc4Fx6J3i4D8ezZs3L46YiPpj4PADDM3RMrlvvi0K/bIZMJ+Pt0GHr3GaQwr2v2rMmwrlcH+fn5SEi4DreBHvLligTRDROUeJ+BrKwX41NnzpzB9OnT4evriw4dOgAAzp07h/nz52Px4sXo1atXqTvCfQboZRVxnwEiKj9lvs/Arrlqa8to4AK1tVVWSpwZqFatmsJWw4Ig4Ouvv5aX/RdTuLi4yMciiYiISPuVOBg4depUWfaDiIhIe1SQzYLUpcTBQLdu3cqyH0RERNpDZHMG3mpNUU5ODq5cuYLY2FiFg4iIqELT4NLCdevWoX79+jA0NISjoyMiIiJeW3/VqlWwsbGBkZER6tatiylTppR6YrBKqwnS09Ph7u6O3377rdj3OWeAiIio9IKCguDl5YXAwEA4Ojpi1apVcHZ2RkJCAszNiy41/+mnnzBz5kxs3boVnTp1wtWrVzFs2DBIJBIEBASU+LoqZQYmT56MR48eITw8HEZGRjh+/Di2b9+OJk2a4NCh4p/GRUREVGFo6BHGAQEBGDVqFNzd3dG8eXMEBgaiUqVK2Lp1a7H1Q0ND0blzZwwYMAD169fHRx99BDc3tzdmE5SpFAz8+eefCAgIgIODA3R0dGBtbY1BgwZh6dKl8Pf3V6VJIiIi7aHGYCA3NxdZWVkKh/LD+gAgLy8PkZGRcHJykpfp6OjAyckJYWFhxXazU6dOiIyMlP/yv3HjBo4dO1bqJf4qBQPZ2dnydEX16tWRnl64J3arVq0QFcU9zomIiP5T3MP5ivvinJGRgYKCAlhYWCiUW1hYICUlpdi2BwwYgPnz56NLly7Q19dHo0aN8MEHH2DWrFnF1n8VlYIBGxsbJCQkAABat26N77//Hvfu3UNgYCBq1aqlSpNERETaQ5Cp7fD29kZmZqbC4e3trZZuhoSE4Ntvv8X69esRFRWFX375BUePHsWCBaXb6EilCYSenp5ITk4GAPj4+ODjjz/Gzp07YWBggO3bt6vSJBERkdYQZOp7wJBUKi3Rw/hMTU2hq6uL1NRUhfLU1FRYWloWe87cuXMxePBgjBw5EkBhhj47OxujR4/G7NmzS/wgMpWCgUGDBsn/bG9vj1u3buHKlSuoV68eTE1NX3MmERERFcfAwAD29vYIDg5G3759ARQ+gTQ4OBgTJkwo9pycnJwiv/B1dXUBvNgZuCRKHAx4eXmVuNHSLGcgIiLSOhradMjLywtDhw6Fg4MD2rdvj1WrViE7Oxvu7u4AgCFDhqB27dryOQcuLi4ICAhAmzZt4OjoiOvXr2Pu3LlwcXGRBwUlUeJg4OLFiyWq9/LzC4iIiCokDW1H3K9fP6Snp2PevHlISUmBnZ0djh8/Lp9UePv2bYVMwJw5cyCRSDBnzhzcu3cPZmZmcHFxwaJFi0p13RI/tbCs8amF9DI+tZCIXqesn1qYs2Gi2tqq5LFWbW2VFZXmDBAREb3T1DiBsCJgMEBERKRMZA8qYjBARESkTGTBwFs9tZCIiIgqPmYGiIiIlGnH3Ppyw2CAiIhIGYcJiIiISEyYGSAiIlLGpYVEREQip6EdCDWFwwREREQix8wAERGRMg4TEBERiZvA1QREREQkJswMEBERKeMwARERkciJbDUBgwEiIiJlIssMcM4AERGRyDEzQEREpExkqwkYDBARESnjMAERERGJCTMDREREyriagIiISOQ4TEBERERiwswAERGRErE9m4DBABERkTIOExAREZGYMDNARESkTGSZAQYDREREyri0kIiISORElhngnAEiIiKRY2aAiIhIiSCyzACDASIiImUiCwY4TEBERCRyzAwQEREp4w6EREREIsdhAiIiIhITZgaIiIiUiSwzwGCAiIhIiSCIKxjgMAEREZHIMTNARESkjMMEREREIsdggIiISNy4HbGGGFl11XQXSIs8vX9a010gLTLVwVvTXSB6p2lNMEBERKQ1mBkgIiISOXHtRsylhURERGLHzAAREZESTiAkIiISO5EFAxwmICIiEjlmBoiIiJSJbAIhgwEiIiIlYpszwGECIiIikWNmgIiISBmHCYiIiMRNbMMEDAaIiIiUiSwzwDkDREREIsfMABERkRJBZJkBBgNERETKRBYMcJiAiIhI5JgZICIiUsJhAiIiIrETWTDAYQIiIiKRY2aAiIhIidiGCZgZICIiUiLI1HeU1rp161C/fn0YGhrC0dERERERr63/6NEjjB8/HrVq1YJUKkXTpk1x7NixUl2TmQEiIiIlmsoMBAUFwcvLC4GBgXB0dMSqVavg7OyMhIQEmJubF6mfl5eHnj17wtzcHPv27UPt2rVx69YtVKtWrVTXZTBARESkJQICAjBq1Ci4u7sDAAIDA3H06FFs3boVM2fOLFJ/69atePDgAUJDQ6Gvrw8AqF+/fqmvy2ECIiIiZYJEbUdubi6ysrIUjtzc3CKXzMvLQ2RkJJycnORlOjo6cHJyQlhYWLHdPHToEDp27Ijx48fDwsICLVu2xLfffouCgoJS3S6DASIiIiXqnDPg7++PqlWrKhz+/v5FrpmRkYGCggJYWFgolFtYWCAlJaXYft64cQP79u1DQUEBjh07hrlz52LFihVYuHBhqe6XwwRERERlyNvbG15eXgplUqlULW3LZDKYm5tj48aN0NXVhb29Pe7du4dly5bBx8enxO0wGCAiIlIiyCRqa0sqlZbol7+pqSl0dXWRmpqqUJ6amgpLS8tiz6lVqxb09fWhq6srL3vvvfeQkpKCvLw8GBgYlKiPHCYgIiJSoomlhQYGBrC3t0dwcLC8TCaTITg4GB07diz2nM6dO+P69euQyV5c6OrVq6hVq1aJAwGAwQAREZHW8PLywqZNm7B9+3bEx8fDw8MD2dnZ8tUFQ4YMgbe3t7y+h4cHHjx4AE9PT1y9ehVHjx7Ft99+i/Hjx5fquhwmICIiUiII6hsmKI1+/fohPT0d8+bNQ0pKCuzs7HD8+HH5pMLbt29DR+fF9/i6devi999/x5QpU2Bra4vatWvD09MTM2bMKNV1JYIgCGq9ExXpGdTWdBdIizy9f1rTXSAtMtXB+82VSFTWJAWVaft3HXuora064X+qra2ywmECIiIikeMwARERkRJ1riaoCBgMEBERKdGOAfTyw2CAiIhIidgyA5wzQEREJHLMDBARESkRW2aAwQAREZESsc0Z4DABERGRyDEzQEREpITDBERERCKnqe2INYXDBERERCLHzAAREZGS0jx6+F3AYICIiEiJjMMEREREJCYqZQbOnz8PmUwGR0dHhfLw8HDo6urCwcFBLZ0jIiLSBE4gLIHx48fjzp07Rcrv3buH8ePHv3WniIiINEmQSdR2VAQqZQbi4uLQtm3bIuVt2rRBXFzcW3eKiIhIk7gDYQlIpVKkpqYWKU9OToaeHuckEhERVSQqBQMfffQRvL29kZmZKS979OgRZs2ahZ49e6qtc0RERJrAYYISWL58Od5//31YW1ujTZs2AIDo6GhYWFhgx44dau0gERFReRPb0kKVgoHatWsjNjYWu3btQkxMDIyMjODu7g43Nzfo6+uru49ERERUhlQe4K9cuTJGjx6tzr4QERFpBbEtLVQ5GEhMTMSqVasQHx8PAGjevDk8PT3RqFEjtXWOiIhIE7iaoAR+//13NG/eHBEREbC1tYWtrS3Cw8PRokULnDx5Ut19JCIiojKkUjAwc+ZMTJkyBeHh4QgICEBAQADCw8MxefJkzJgxQ9191Gq+PtNw51YUHmdex++/7Ubjxg3eeI7H2KG4fvUcnmQlIvTMYbRzsFN4XyqVYs3qRUhNvoxHD65iT9BGmJubKtRxsG+NE8eDkJEWh/TUf3DsyC7Y2jZXqPNRz244e/owHv6bgOR7sdgTtBHW1nXe+p6pbF2IvoTx033Q3XUgWnb+BMF/h2q6S1RGug7+CD5n1mJFwg54HVyIeq1fnVnt2L8HPPf4YnHMFiyO2YLxO+cUqW9QSYov/dwxP2w9ll/ZgVknV6DzQKeyvo13kkyQqO2oCFQKBuLj4zFixIgi5cOHDxfVpkPfTBuHCeOHY9yEmejUxQXZOTk4dmQXpFLpK8/56itXLF/mgwULA9DO8WPExMbh2NFdMDOrKa+zYrkv+vTuif5uY9Djwy9gVcsS+/Zslr9fuXIlHD2yC7fv3EOnLi7o1v0zPH6SjWNHdsn3eahfvy5+2b8Vp0LOwr7dR+jVewBMa9bA3pfaIe309Okz2DRuiNlTx2m6K1SG2vTpiM/mDMHx1fuxrPdM3Iu7hXE/zkKVmibF1m/SoQUiD4Virdt8BHw+Fw+T/8W4HbNR1aK6vM5nc4bgvW52+HHKd/jWyQshW4/hS7/haOlkX1639c4QBInajopApWDAzMwM0dHRRcqjo6Nhbm7+tn2qMCZNHIlv/Vfj8OETuHQpHsPcPWFlZYFPP3V+5TlTPEdh85afsP3HPYiPv4Zx42ciJ+cp3If1BwCYmBhjuHt/TJvuh1MhZxF18RJGjJqCTp3awbF94a6PzWwao2bN6vD1W46rVxMRF3cVCxYGwNLSXP7Nv21bW+jq6mLuvCW4ceMWLkZfRsDK72HXugU3htJyXTu2w6TRQ+HUrbOmu0JlqPvI3gjdHYzwvSFIuX4Pe2ZvRt7TPHT4unux9X+cvBZndp7AvbhbSEu8j59nBEJHIkHTzq3kdRrY2yBi/1+4fi4OD+6mI/TnYNyPvwXr1o3L67aoglIpGBg1ahRGjx6NJUuW4PTp0zh9+jQWL16MMWPGYNSoUeruo1Zq0KAeatWyQPCfZ+RlWVmPERFxER0ci4/C9fX10batLYL/PC0vEwQBwX+eQYcOhefYt7WFgYEBgoNf1ElISMStW3fldRKuJiIj4wGGu/eHvr4+DA0N4T7MDXHxV5GUVPjMiKioWMhkMgwb2g86OjowMTHGwIFfIDj4NPLz89X+8yCiktPV10Xdlg2RcPaSvEwQBCScvYQGbZuUqA0DIyl09PWQ8+iJvOxmZAJaOjnIswVNOraAWYNauHI6Vr03IAKCoL6jIlDpK+LcuXNhbGyMFStWwNvbGwBgZWUFX19fTJo06Y3n5+bmIjc3V6FMEARIJBUjnQIAlhaFGZDU1HSF8tS0DFhaFp8dMTWtAT09PaSlZiiUp6Wlo5lN4difhaUZcnNzkZmZVaSOpaUZAODJk2x82PNL7N+7BbNnTQYAXLt+E716D0BBQQEAICnpDj7pNQA//xSIDeuXQE9PD2FhF9DHdfDb3TgRvbXK1U2gq6eLxxmZCuWP0zNh0ciqRG24zhyIrNQHCgHFft8f0M9/NBaEB6LgeT4EmYCfvTciMSJerf0Xg4oy1q8upc4M5OfnY8eOHRgwYADu3r2LzMxMZGZm4u7du/D09CzRL3R/f39UrVpV4RBkj1W6gfLi5vYZHj24Kj/09TWXajc0NMSm75cjNOwCOndxwfvd+uKffxJw6NcfYWhoCACwsDBDYOAy7Ni5Fx069UL3Hp8jLy8Pe3Zv1Fi/iUg9nDw+RVuXTtg8ZgXyc5/Ly98f+jHq2zXBxhFLsMzFGwcW7cBX84crDCVQyYhtzkCpf6Pp6elh7Nix8v0FjI2NS31Rb29veHl5KZRVr9ms1O2Up8OHTyAi4qL8tVRqAKDwl25KSpq83MLcFNEx/xTbRkbGA+Tn58PcQnFlgLm5GVL+n2FITUmHVCpF1aomCtkBc3MzpKQU1nHr3xfW1nXRuasrhP/noAYNHo+MtDi4un6EPXsOYZzHMGRmZmGm9yJ5G0OGTcKtmxfg2L4twiOi3ubHQURvIfthFgryC2BsWlWh3NisKh6nP3rtuT1G9YGTx6dYN3Ah7l+5LS/Xl+qjzzdu2DxmOeJOFf5bdf/KbdRpXh8fju6Dqy9lEIiUqTRnoH379rh48eKbK76CVCqFiYmJwqHtQwRPnmQjMTFJfsTFXUVycip6dO8ir2NsXAXt27fBufDIYtt4/vw5oqJiFc6RSCTo0b0Lzp0rPCcyKhZ5eXno0eNFnaZNG8Hauo68TqVKRpDJZPJAAID8tY5O4f/SSkZGEGQyhev/N4TwXx0i0oyC5wW4c/kGmnZ68Y1dIpHAplNL3Iy69srzPhzjCueJXyBwqD/uXLqh8J6uvh70DPQU/l0ACv9t0PZ/X7WR2JYWqpTrHjduHKZOnYq7d+/C3t4elStXVnjf1tZWLZ3TdmvWbsYs70m4dv0GkpLuwM/3G9y/n4pff/1dXufE8SAc/PU3rN+wDQCwcvUm/LBlJSKjYnH+/EVMmjgKlSsbYdv2IACFkxC3/rAby5f64OGDR8jKeozVqxYiLOyC/Nv8H8F/Y8niOVi75lusW78VOjo6mP7NBOTn5yMkpHBN+rHfguHpOQpzZk/G7qBfYVylMhYumImkpDu4GH25fH9QVCo5OU9x++59+et791Nx5WoiqpoYo9Yr5qNQxXNq81EMWjEOdy4l4lZ0Ij4Y0QsGlaQI3xsCABi0YjwyUx/g8NKfAQBOY13Ra8rX2O65Bv/eTYOxWWFWITf7GfJycvHsyVNcO/cPPvUehOfP8vDgbjoad2iOdp+/j4MLf9TUbVZYFWTen9qoFAz071+4DO7lyYISiUQ+CfC/b6DvumXL16Ny5UoIXL8U1aqZ4OzZ8+jtMkhhcmTDhtYwNa0hf7137yGYmdaA77xpsLQ0Q0zMP+jdZxDS0l5MKpw6zRcymQx7gjZCKpXixMkQTJg4S/5+QkIi+n42DHPneOHM34cgk8kQHV3Yzn9DFqdCzmLQkPGYNnUcpk0dh5ycpzgXHoneLgPx7NmzcvjpkKouX7mG4RNfbN61dG3hPI9PP3HCojlTNdUtUrOLR8JQpYYJek35GiZm1XA3PgkbhvrLJxVWr10TgvAiu9d5UE/oSfUxIlDxM/Dbqr34bdU+AMC2iavhMn0AhqyaiErVquDhvXQcXbYbZ3ZyZ1h6PYmgnFMqgVu3br32fWtr61J3RM+gdqnPoXfX0/un31yJRGOqg7emu0BaZk1SUJm2H1rrC7W11Sl5v9raKisqZQZU+WVPRERUUVSUVQDqovL6uISEBKxdu1a+quC9997DxIkTYWNjo7bOERERUdlTaVr5/v370bJlS0RGRqJ169Zo3bo1oqKi0LJlS+zfr/3pECIioteRqfGoCFTKDEyfPh3e3t6YP3++QrmPjw+mT5+OL75Q31gLERFReRMgrmEClTIDycnJGDJkSJHyQYMGITk5+a07RUREROVHpWDggw8+wOnTRWd7nzlzBl27dn3rThEREWmSTFDfURGoNEzg6uqKGTNmIDIyEh06dAAAnDt3Dnv37oWfnx8OHTqkUJeIiKgikYlsmEClfQZKup1taTYg4j4D9DLuM0Av4z4DpKys9xkItuintrY+TC3bvqqDSpkBmayizI8kIiKiN1F5n4Hg4GAEBwcjLS1NITiQSCTYsmWLWjpHRESkCWL7yqtSMODn54f58+fDwcEBtWrV4hOxiIjonSK2pYUqBQOBgYHYtm0bBg8erO7+EBERUTlTKRjIy8tDp06d1N0XIiIirSC2YQKV9hkYOXIkfvrpJ3X3hYiISCtwO+ISePbsGTZu3Ig//vgDtra20NfXV3g/ICBALZ0jIiKisqdSMBAbGws7OzsAwOXLlxXe42RCIiKq6DiBsAROnTql7n4QERFpDZm4YgHV5gwQERHRu0PlTYeIiIjeVWJ7NgGDASIiIiUV5GGDasNggIiISElFWRKoLpwzQEREJHLMDBARESmRiWyZPIMBIiIiJWKbM8BhAiIiIpFjZoCIiEiJ2CYQMhggIiJSwh0IiYiISFQYDBARESmRQaK2o7TWrVuH+vXrw9DQEI6OjoiIiCjRebt374ZEIkHfvn1LfU0GA0REREoENR6lERQUBC8vL/j4+CAqKgqtW7eGs7Mz0tLSXnteUlISpk2bhq5du5byioUYDBAREZWh3NxcZGVlKRy5ubnF1g0ICMCoUaPg7u6O5s2bIzAwEJUqVcLWrVtf2X5BQQEGDhwIPz8/NGzYUKU+MhggIiJSIpOo7/D390fVqlUVDn9//yLXzMvLQ2RkJJycnORlOjo6cHJyQlhY2Cv7On/+fJibm2PEiBEq3y9XExARESlR59JCb29veHl5KZRJpdIi9TIyMlBQUAALCwuFcgsLC1y5cqXYts+cOYMtW7YgOjr6rfrIYICIiEiJOncglEqlxf7yf1uPHz/G4MGDsWnTJpiamr5VWwwGiIiItICpqSl0dXWRmpqqUJ6amgpLS8si9RMTE5GUlAQXFxd5mUxWmNPQ09NDQkICGjVqVKJrc84AERGREnXOGSgpAwMD2NvbIzg4+EU/ZDIEBwejY8eOReo3a9YMly5dQnR0tPxwdXVF9+7dER0djbp165b42swMEBERKdHUdsReXl4YOnQoHBwc0L59e6xatQrZ2dlwd3cHAAwZMgS1a9eGv78/DA0N0bJlS4Xzq1WrBgBFyt+EwQAREZGW6NevH9LT0zFv3jykpKTAzs4Ox48fl08qvH37NnR01J/UlwiCoBVPatQzqK3pLpAWeXr/tKa7QFpkqoO3prtAWmZNUlCZtv99nUFqa2vM3Z1qa6usMDNARESkROCDioiIiEhMmBkgIiJSoqkJhJrCYICIiEiJ2IIBDhMQERGJHDMDRERESrRimV05YjBARESkpDQ7B74LGAwQEREp4ZwBIiIiEhVmBoiIiJSILTPAYICIiEiJ2CYQcpiAiIhI5JgZICIiUsLVBERERCIntjkDHCYgIiISOWYGiIiIlIhtAiGDASIiIiUykYUDDAZIK0118NZ0F0iLrLjgr+kuEL3TGAwQEREpEdsEQgYDRERESsQ1SMBggIiIqAixZQa4tJCIiEjkmBkgIiJSwh0IiYiIRE5sSws5TEBERCRyzAwQEREpEVdegMEAERFREVxNQERERKLCzAAREZESsU0gZDBARESkRFyhAIcJiIiIRI+ZASIiIiVim0DIYICIiEgJ5wwQERGJnLhCAc4ZICIiEj1mBoiIiJRwzgAREZHICSIbKOAwARERkcgxM0BERKSEwwREREQiJ7alhRwmICIiEjlmBoiIiJSIKy/AYICIiKgIDhMQERGRqDAzQEREpISrCYiIiERObJsOMRggIiJSIrbMAOcMEBERiRwzA0REREo4TEBERCRyHCYgIiIiUWFmgIiISIlM4DABERGRqIkrFOAwARERkegxM0BERKREbM8mYDBARESkRGxLCzlMQEREJHLMDBARESkR2z4DDAaIiIiUcM4AERGRyHHOABEREYmKSpmBNm3aQCKRFCmXSCQwNDRE48aNMWzYMHTv3v2tO0hERFTexDZnQKXMwMcff4wbN26gcuXK6N69O7p3744qVaogMTER7dq1Q3JyMpycnPDrr7+qu79ERERlThAEtR0VgUrBQEZGBqZOnYrTp09jxYoVWLFiBf7++29MmzYN2dnZOHHiBObMmYMFCxaou79ERETvtHXr1qF+/fowNDSEo6MjIiIiXll306ZN6Nq1K6pXr47q1avDycnptfVfRaVgYM+ePXBzcytS3r9/f+zZswcA4ObmhoSEBFWaJyIi0igZBLUdpREUFAQvLy/4+PggKioKrVu3hrOzM9LS0oqtHxISAjc3N5w6dQphYWGoW7cuPvroI9y7d69U11UpGDA0NERoaGiR8tDQUBgaGgIAZDKZ/M9EREQViUyNR25uLrKyshSO3NzcYq8bEBCAUaNGwd3dHc2bN0dgYCAqVaqErVu3Flt/165dGDduHOzs7NCsWTNs3rwZMpkMwcHBpbpflYKBiRMnYuzYsfD09MTOnTuxc+dOeHp6wsPDA5MmTQIA/P7777Czs1OleSIioneGv78/qlatqnD4+/sXqZeXl4fIyEg4OTnJy3R0dODk5ISwsLASXSsnJwfPnz9HjRo1StVHlVYTzJkzBw0aNMB3332HHTt2AABsbGywadMmDBgwAAAwduxYeHh4qNI8ERGRRqlznwFvb294eXkplEml0iL1MjIyUFBQAAsLC4VyCwsLXLlypUTXmjFjBqysrBQCipJQedOhgQMHYuDAga9838jISNWmiYiINEqdOxBKpdJif/mr2+LFi7F7926EhISUepieOxASERFpAVNTU+jq6iI1NVWhPDU1FZaWlq89d/ny5Vi8eDH++OMP2NralvraKs0ZKCgowPLly9G+fXtYWlqiRo0aCgcREVFFpol9BgwMDGBvb68w+e+/yYAdO3Z85XlLly7FggULcPz4cTg4OKh0vyoFA35+fggICEC/fv2QmZkJLy8vfP7559DR0YGvr69KHSEiItIW6lxNUBpeXl7YtGkTtm/fjvj4eHh4eCA7Oxvu7u4AgCFDhsDb21tef8mSJZg7dy62bt2K+vXrIyUlBSkpKXjy5EmprqvSMMGuXbuwadMm9O7dG76+vnBzc0OjRo1ga2uLc+fOyVcUEBERVUSaelBRv379kJ6ejnnz5iElJQV2dnY4fvy4fFLh7du3oaPz4nv8hg0bkJeXhy+//FKhHR8fn1J9OVcpM5CSkoJWrVoBAKpUqYLMzEwAQJ8+fXD06FFVmqywfH2m4c6tKDzOvI7ff9uNxo0bvPEcj7FDcf3qOTzJSkTomcNo52Cn8L5UKsWa1YuQmnwZjx5cxZ6gjTA3Ny3SzpDBXyMq8iSeZCXi/t0YrFm9SP6etXUd5OfdK3I4tm/71vdMqus6+CP4nFmLFQk74HVwIeq1bvTKuh3794DnHl8sjtmCxTFbMH7nnCL1DSpJ8aWfO+aHrcfyKzsw6+QKdB5YulnEpP0uRF/C+Ok+6O46EC07f4Lgv4vu80LvjgkTJuDWrVvIzc1FeHg4HB0d5e+FhIRg27Zt8tdJSUnFDk2UNkuvUjBQp04dJCcnAwAaNWqEEydOAADOnz9fLjMmtcU308ZhwvjhGDdhJjp1cUF2Tg6OHdn12p/BV1+5YvkyHyxYGIB2jh8jJjYOx47ugplZTXmdFct90ad3T/R3G4MeH34Bq1qW2Ldns0I7kz1HY8H86Vi6bB1s7XrA+ZP+OHEypMj1PnLuh9p17eRHZFSs2u6fSqdNn474bM4QHF+9H8t6z8S9uFsY9+MsVKlpUmz9Jh1aIPJQKNa6zUfA53PxMPlfjNsxG1UtqsvrfDZnCN7rZocfp3yHb528ELL1GL70G46WTvbldVtUDp4+fQabxg0xe+o4TXdFNDS1A6GmqBQMfPbZZ/IJDhMnTsTcuXPRpEkTDBkyBMOHD1drB7XZpIkj8a3/ahw+fAKXLsVjmLsnrKws8Omnzq88Z4rnKGze8hO2/7gH8fHXMG78TOTkPIX7sP4AABMTYwx3749p0/1wKuQsoi5ewohRU9CpUzv5t/pq1apivt90uA+fjN27D+LGjVu4dCkeR46cLHK9fx88RGpquvzIz88vmx8GvVH3kb0RujsY4XtDkHL9HvbM3oy8p3no8HXxT/f8cfJanNl5AvfibiEt8T5+nhEIHYkETTu3ktdpYG+DiP1/4fq5ODy4m47Qn4NxP/4WrFs3Lq/bonLQtWM7TBo9FE7dOmu6K6LBBxWVwOLFizFr1iwAheMbf//9Nzw8PLBv3z4sXrxYrR3UVg0a1EOtWhYI/vOMvCwr6zEiIi6ig2Px38r09fXRtq0tgv88LS8TBAHBf55Bhw6F59i3tYWBgQGCg1/USUhIxK1bd+V1nJzeh46OBFa1LXEpNgRJNy7g558CUaeOVZFrHtj/A+7fjcFfpw6gT5+earl3Kj1dfV3UbdkQCWcvycsEQUDC2Uto0LZJidowMJJCR18POY9eTAy6GZmAlk4O8mxBk44tYNagFq6cZgaIiEpOLfsMdOzY8bXLHt5FlhbmAIDU1HSF8tS0DFhamhd7jqlpDejp6SEtNUOhPC0tHc1sCseCLSzNkJubi8zMrCJ1LC3NAAANG9SDjo4OZs6YCC+vecjMfIz5ftNx/Lef0aatE54/f44nT7Ix7Rs/hIaeh0wmw+ef98Iv+7bi8y+HF5tBoLJVuboJdPV08TgjU6H8cXomLBoVDeKK4zpzILJSHygEFPt9f0A//9FYEB6Iguf5EGQCfvbeiMSIeLX2n0hsKkp6X11KHAwcOnQIn3zyCfT19XHo0KHX1nV1dX3t+7m5uUUe0iAIAiQSSUm7U+7c3D7DhnVL5K9dPx2isb7o6OjAwMAAU6bMxck//gYADBw8DvfuRKP7B51w4uRf+Pffh1i1eqP8nAuRMahVyxLTvDwYDFRATh6foq1LJ6zt74f83Ofy8veHfoz6dk2wccQSPLiXgUbt38NX84cjM/Uhrr4UNBBR6WhqNYGmlDgY6Nu3L1JSUmBubo6+ffu+sp5EIkFBQcFr2/L394efn5/ieTpVINEtfiKVNjh8+AQiIi7KX0ulBgAACwszpKS8eLSkhbkpomP+KbaNjIwHyM/Ph7mF4soAc3MzpPw/w5Cakg6pVIqqVU0UsgPm5mZISSmsk5xcuDtVXPw1hbYzMh6gbt3ar7yHiIgoOH3YtUT3S+qV/TALBfkFMDatqlBubFYVj9MfvfbcHqP6wMnjU6wbuBD3r9yWl+tL9dHnGzdsHrMccacKP5v3r9xGneb18eHoPgwGiKjESjxnQCaTwdzcXP7nVx1vCgSAwoc2ZGZmKhwSHWPV76IcPHmSjcTEJPkRF3cVycmp6NG9i7yOsXEVtG/fBufCI4tt4/nz54iKilU4RyKRoEf3Ljh3rvCcyKhY5OXloUePF3WaNm0Ea+s68jqhYRcAADZNXywzq169GkxNa+D27buvvIfWrVsgOaX4Z2JT2Sp4XoA7l2+gaacXk/8kEglsOrXEzahrrzzvwzGucJ74BQKH+uPOpRsK7+nq60HPQK/IBCWZTKbVWTaiikAmCGo7KgKNPJuguIc2VMR/vNas3YxZ3pNw7foNJCXdgZ/vN7h/PxW//vq7vM6J40E4+OtvWL9hGwBg5epN+GHLSkRGxeL8+YuYNHEUKlc2wrbtQQAKJyFu/WE3li/1wcMHj5CV9RirVy1EWNgFhEdEAQCuXbuBXw8dR0CAHzw8ZiDr8WMsWuiNKwnXcSqkcP3x4MFfIS8vD9HRlwEAn/XtBfdh/TF6zLRy/AnRy05tPopBK8bhzqVE3IpOxAcjesGgkhThe0MAAINWjEdm6gMcXvozAMBprCt6Tfka2z3X4N+7aTA2K8wq5GY/Q15OLp49eYpr5/7Bp96D8PxZHh7cTUfjDs3R7vP3cXDhj5q6TSoDOTlPcfvuffnre/dTceVqIqqaGKPWK+Yo0dupGL/C1afEwcCaNWtK3KhYdiBctnw9KleuhMD1S1GtmgnOnj2P3i6DFOZDNGxoDVPTF89r2Lv3EMxMa8B33jRYWpohJuYf9O4zCGlpLyYVTp3mC5lMhj1BGyGVSnHiZAgmTJylcO1h7p5YsdwXh37dDplMwN+nw9C7zyCFpYOzZ02Gdb06yM/PR0LCdbgN9MAvv4hrUyhtcvFIGKrUMEGvKV/DxKwa7sYnYcNQf/mkwuq1a0IQXmxe2nlQT+hJ9TEicKpCO7+t2ovfVu0DAGybuBou0wdgyKqJqFStCh7eS8fRZbtxZifnhbxLLl+5huETZ8hfL11bOB/o00+csGjO1FedRlRiEqGEiyAbNFDcWS89PR05OTmoVq0aAODRo0eoVKkSzM3NcePGjWJaeD09g1ePdZP4jLPq8uZKJBorLvhrugukZfRNG5Zp+51r91BbW2fv/am2tspKiecM3Lx5U34sWrQIdnZ2iI+Px4MHD/DgwQPEx8ejbdu2WLBgQVn2l4iIqMxxB8ISmDt3LtauXQsbGxt5mY2NDVauXIk5c+aorXNERESawB0ISyA5ObnYbW0LCgqQmpr61p0iIiKi8qNSMPDhhx9izJgxiIqKkpdFRkbCw8MDTk58YhoREVVsHCYoga1bt8LS0hIODg7yZYLt27eHhYUFNm/e/OYGiIiItJigxv8qApX2GTAzM8OxY8dw7do1xMcX7oHerFkzNG3aVK2dIyIiorL3VpsONWnSBE2aNEFBQQEuXbqEhw8fonr16m8+kYiISItVlIl/6qLSMMHkyZOxZcsWAIWTBrt164a2bduibt26CAkJUWf/iIiIyh3nDJTAvn370Lp1awDA4cOHcePGDVy5cgVTpkzB7Nmz1dpBIiIiKlsqBQMZGRmwtLQEABw7dgxff/01mjZtiuHDh+PSJT4pjYiIKjbuM1ACFhYWiIuLQ0FBAY4fP46ePXsCAHJycqCrq6vWDhIREZU3sQ0TqDSB0N3dHV9//TVq1aoFiUQi31sgPDwczZo1U2sHiYiIqGypFAz4+vqiVatWuH37Nr7++mv544h1dXXh7e2t1g4SERGVt4qyP4C6qLy0sFq1aoiMjISPjw9kMpnCe66urm/dMSIiIk2RVZCxfnVRKRjw8/PD/Pnz4eDgIB8qICIielcwM1ACgYGB2LZtGwYPHqzu/hAREVE5UykYyMvLQ6dOndTdFyIiIq0gtmEClZYWjhw5Ej/99JO6+0JERKQV+KCiEnj27Bk2btyIP/74A7a2ttDX11d4PyAgQC2dIyIiorKnUjAQGxsLOzs7AMDly5cV3uNkQiIiqujENkygUjBw6tQpdfeDiIhIa1SU9L66qDRngIiIiN4dKm86RERE9K7iMAEREZHIcZiAiIiIRIWZASIiIiWCIHtzpXcIgwEiIiIlMpENEzAYICIiUiKIbAIh5wwQERGJHDMDRERESjhMQEREJHIcJiAiIiJRYWaAiIhICXcgJCIiEjnuQEhERESiwswAERGRErFNIGQwQEREpERsSws5TEBERCRyzAwQEREp4TABERGRyHFpIRERkciJLTPAOQNEREQix8wAERGRErGtJmAwQEREpITDBERERCQqzAwQEREp4WoCIiIikeODioiIiEhUmBkgIiJSwmECIiIikeNqAiIiIhIVZgaIiIiUiG0CIYMBIiIiJRwmICIiEjlBENR2lNa6detQv359GBoawtHREREREa+tv3fvXjRr1gyGhoZo1aoVjh07VuprMhggIiLSEkFBQfDy8oKPjw+ioqLQunVrODs7Iy0trdj6oaGhcHNzw4gRI3Dx4kX07dsXffv2xeXLl0t1XYmgJbkQPYPamu4CaZFxVl003QXSIisu+Gu6C6Rl9E0blmn76vydlP34BnJzcxXKpFIppFJpkbqOjo5o164dvvvuOwCATCZD3bp1MXHiRMycObNI/X79+iE7OxtHjhyRl3Xo0AF2dnYIDAwseScF0hrPnj0TfHx8hGfPnmm6K6QF+Hmgl/HzUHH5+PgIABQOHx+fIvVyc3MFXV1d4cCBAwrlQ4YMEVxdXYttu27dusLKlSsVyubNmyfY2tqWqo8cJtAiubm58PPzKxJBkjjx80Av4+eh4vL29kZmZqbC4e3tXaReRkYGCgoKYGFhoVBuYWGBlJSUYttOSUkpVf1X4WoCIiKiMvSqIQFtwswAERGRFjA1NYWuri5SU1MVylNTU2FpaVnsOZaWlqWq/yoMBoiIiLSAgYEB7O3tERwcLC+TyWQIDg5Gx44diz2nY8eOCvUB4OTJk6+s/yocJtAiUqkUPj4+Wp9OovLBzwO9jJ8HcfDy8sLQoUPh4OCA9u3bY9WqVcjOzoa7uzsAYMiQIahduzb8/QtX2Hh6eqJbt25YsWIFevfujd27d+PChQvYuHFjqa6rNUsLiYiICPjuu++wbNkypKSkwM7ODmvWrIGjoyMA4IMPPkD9+vWxbds2ef29e/dizpw5SEpKQpMmTbB06VL06tWrVNdkMEBERCRynDNAREQkcgwGiIiIRI7BABERkcgxGFCzDz74AJMnTy5R3ZCQEEgkEjx69Oitrlm/fn2sWrXqrdogIu2m/PdcIpHg4MGDGusPvVu4tJCIqAJKTk5G9erVNd0NekcwGCCqIJ4/fw59fX1Nd4O0RGl3mCN6HQ4TlKEdO3bAwcEBxsbGsLS0xIABA4p9JvXZs2dha2sLQ0NDdOjQochzqM+cOYOuXbvCyMgIdevWxaRJk5CdnV1et0Eq+OCDDzBp0iRMnz4dNWrUgKWlJXx9feXvSyQSbNiwAZ988gmMjIzQsGFD7Nu3T/5+UlISJBIJgoKC0K1bNxgaGmLXrl2QyWSYP38+6tSpA6lUCjs7Oxw/flzh2nfv3oWbmxtq1KiBypUrw8HBAeHh4eV16/R/r/sMCIIAX19f1KtXD1KpFFZWVpg0aZL83LS0NLi4uMDIyAgNGjTArl27irSvPEwwY8YMNG3aFJUqVULDhg0xd+5cPH/+XOGchQsXwtzcHMbGxhg5ciRmzpwJOzu7srh9qmAYDJSh58+fY8GCBYiJicHBgweRlJSEYcOGFan3zTffYMWKFTh//jzMzMzg4uIi/0ucmJiIjz/+GF988QViY2MRFBSEM2fOYMKECeV8N1Ra27dvR+XKlREeHo6lS5di/vz5OHnypPz9uXPn4osvvkBMTAwGDhyI/v37Iz4+XqGNmTNnwtPTE/Hx8XB2dsbq1auxYsUKLF++HLGxsXB2doarqyuuXbsGAHjy5Am6deuGe/fu4dChQ4iJicH06dMhk8nK9d6p0Ks+A/v378fKlSvx/fff49q1azh48CBatWolP2/YsGG4c+cOTp06hX379mH9+vXFfpF4mbGxMbZt24a4uDisXr0amzZtwsqVK+Xv79q1C4sWLcKSJUsQGRmJevXqYcOGDWV271TBlOqBx/RG3bp1Ezw9PYt97/z58wIA4fHjx4IgCMKpU6cEAMLu3bvldf7991/ByMhICAoKEgRBEEaMGCGMHj1aoZ3Tp08LOjo6wtOnTwVBEARra+siz7MmzerWrZvQpUsXhbJ27doJM2bMEARBEAAIY8eOVXjf0dFR8PDwEARBEG7evCkAEFatWqVQx8rKSli0aFGRdseNGycIgiB8//33grGxsfDvv/+q9X6o9F73GVixYoXQtGlTIS8vr8h5CQkJAgAhIiJCXhYfHy8AUPh7DqDIc+9ftmzZMsHe3l7+2tHRURg/frxCnc6dOwutW7cu3Y3RO4mZgTIUGRkJFxcX1KtXD8bGxujWrRsA4Pbt2wr1Xn6gRI0aNWBjYyP/hhgTE4Nt27ahSpUq8sPZ2RkymQw3b94sv5uhUrO1tVV4XatWLYVvd8oPEunYsWORzICDg4P8z1lZWbh//z46d+6sUKdz587y86Kjo9GmTRvUqFFDLfdAb+dVn4GvvvoKT58+RcOGDTFq1CgcOHAA+fn5AID4+Hjo6enB3t5efl6zZs1QrVq1114rKCgInTt3hqWlJapUqYI5c+Yo/FuTkJCA9u3bK5yj/JrEi8FAGcnOzoazszNMTEywa9cunD9/HgcOHAAA5OXllbidJ0+eYMyYMYiOjpYfMTExuHbtGho1alRW3Sc1UJ7sJ5FISp2ur1y5cqnqGxkZlao+la1XfQbq1q2LhIQErF+/HkZGRhg3bhzef//9ImP8JRUWFoaBAweiV69eOHLkCC5evIjZs2eX6t8aEjcGA2XkypUr+Pfff7F48WJ07doVzZo1e+WY37lz5+R/fvjwIa5evYr33nsPANC2bVvExcWhcePGRQ4DA4NyuRcqGy//f//v9X//34tjYmICKysrnD17VqH87NmzaN68OYDCb6LR0dF48OCB+jtMamVkZAQXFxesWbMGISEhCAsLw6VLl9CsWTPk5+cjMjJSXjchIeG1+5GEhobC2toas2fPhoODA5o0aYJbt24p1LGxscH58+cVypRfk3hxaWEZqVevHgwMDLB27VqMHTsWly9fxoIFC4qtO3/+fNSsWRMWFhaYPXs2TE1N0bdvXwCFM4Q7dOiACRMmYOTIkahcuTLi4uJw8uRJfPfdd+V4R6Rue/fuhYODA7p06YJdu3YhIiICW7Zsee0533zzDXx8fNCoUSPY2dnhhx9+QHR0tHy2uZubG7799lv07dsX/v7+qFWrFi5evAgrK6tSP9+cys62bdtQUFAAR0dHVKpUCTt37oSRkRGsra1Rs2ZNfPzxxxgzZgw2bNgAPT09TJ48+bVZnyZNmuD27dvYvXs32rVrh6NHj8ozkf+ZOHEiRo0aBQcHB3Tq1AlBQUGIjY1Fw4YNy/p2qQJgZqCMmJmZYdu2bdi7dy+aN2+OxYsXY/ny5cXWXbx4MTw9PWFvb4+UlBQcPnxY/q3f1tYWf/31F65evYquXbuiTZs2mDdvHqysrMrzdqgM+Pn5Yffu3bC1tcWPP/6In3/+Wf4N/1UmTZoELy8vTJ06Fa1atcLx48dx6NAhNGnSBABgYGCAEydOwNzcHL169UKrVq2wePFi6OrqlsctUQlVq1YNmzZtQufOnWFra4s//vgDhw8fRs2aNQEAP/zwA6ysrNCtWzd8/vnnGD16NMzNzV/ZnqurK6ZMmYIJEybAzs4OoaGhmDt3rkKdgQMHwtvbG9OmTUPbtm1x8+ZNDBs2DIaGhmV6r1Qx8BHGRBogkUhw4MABeQaISBN69uwJS0tL7NixQ9NdIQ3jMAERkQjk5OQgMDAQzs7O0NXVxc8//4w//vhDYe8LEi8GA0REIiCRSHDs2DEsWrQIz549g42NDfbv3w8nJydNd420AIcJiIiIRI4TCImIiESOwQAREZHIMRggIiISOQYDREREIsdggIiISOQYDBAREYkcgwEiIiKRYzBAREQkcv8DXd54IWs55KEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# check the correlation each column against the label\n",
    "import seaborn as sns\n",
    "sns.heatmap(df.corr(), annot=True, vmin=0, vmax=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Features Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dropping column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dropping unused column based on the heatmap and pvalue in eda phase\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "samp = pd.read_csv(\"temp_data/1/train_samp_cleaned.csv\")\n",
    "samp.drop([\"id_peserta\", \"pisat\", \"kelasrawat\", \"dati2\", \"typefaskes\", \"cbg\"], axis=1, inplace=True)\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save to some folder\n",
    "samp.to_csv(\"temp_data/1/train_samp_cleaned.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handle nday"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>label</th>\n",
       "      <th>nday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>2018-07-25 17:00:00+00:00</td>\n",
       "      <td>2018-07-25 17:00:00+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2019-05-27 17:00:00+00:00</td>\n",
       "      <td>2019-05-30 17:00:00+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2019-07-16 17:00:00+00:00</td>\n",
       "      <td>2019-07-16 17:00:00+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>2019-10-17 17:00:00+00:00</td>\n",
       "      <td>2019-10-17 17:00:00+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2018-04-18 17:00:00+00:00</td>\n",
       "      <td>2018-04-18 17:00:00+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-30 17:00:00+00:00</td>\n",
       "      <td>2021-01-03 17:00:00+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-30 17:00:00+00:00</td>\n",
       "      <td>2021-01-03 17:00:00+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>2020-12-30 17:00:00+00:00</td>\n",
       "      <td>2021-01-06 17:00:00+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>2020-12-29 17:00:00+00:00</td>\n",
       "      <td>2020-12-31 17:00:00+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-29 17:00:00+00:00</td>\n",
       "      <td>2020-12-31 17:00:00+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  usia jenkel                 tgldatang  \\\n",
       "0           165666         486    48      P 2018-07-25 17:00:00+00:00   \n",
       "1          1010828         520    63      L 2019-05-27 17:00:00+00:00   \n",
       "2           166042         523    53      P 2019-07-16 17:00:00+00:00   \n",
       "3           168937         549    54      P 2019-10-17 17:00:00+00:00   \n",
       "4          1005899         549    53      P 2018-04-18 17:00:00+00:00   \n",
       "...            ...         ...   ...    ...                       ...   \n",
       "11291759   9983563     8527704     0      P 2020-12-30 17:00:00+00:00   \n",
       "11291760  11053870     8527709     0      P 2020-12-30 17:00:00+00:00   \n",
       "11291761   7461049     8527751     0      L 2020-12-30 17:00:00+00:00   \n",
       "11291762   1075162     8527853     0      L 2020-12-29 17:00:00+00:00   \n",
       "11291763    102794     8527876     0      P 2020-12-29 17:00:00+00:00   \n",
       "\n",
       "                         tglpulang  jenispel       biaya  jenispulang  label  \\\n",
       "0        2018-07-25 17:00:00+00:00         2    184300.0          1.0      0   \n",
       "1        2019-05-30 17:00:00+00:00         1  10628400.0          1.0      0   \n",
       "2        2019-07-16 17:00:00+00:00         2    187300.0          1.0      0   \n",
       "3        2019-10-17 17:00:00+00:00         2    187300.0          1.0      0   \n",
       "4        2018-04-18 17:00:00+00:00         2    381600.0          1.0      0   \n",
       "...                            ...       ...         ...          ...    ...   \n",
       "11291759 2021-01-03 17:00:00+00:00         1   3645100.0          1.0      0   \n",
       "11291760 2021-01-03 17:00:00+00:00         1  12273200.0          1.0      0   \n",
       "11291761 2021-01-06 17:00:00+00:00         1   4099000.0          1.0      0   \n",
       "11291762 2020-12-31 17:00:00+00:00         1   3992400.0          1.0      0   \n",
       "11291763 2020-12-31 17:00:00+00:00         1   1624900.0          1.0      0   \n",
       "\n",
       "          nday  \n",
       "0            0  \n",
       "1            3  \n",
       "2            0  \n",
       "3            0  \n",
       "4            0  \n",
       "...        ...  \n",
       "11291759     4  \n",
       "11291760     4  \n",
       "11291761     7  \n",
       "11291762     2  \n",
       "11291763     2  \n",
       "\n",
       "[11291764 rows x 11 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make nday column based on tglpulang-tgldatang in days\n",
    "\n",
    "import pandas as pd\n",
    "df = pd.read_csv(\"temp_data/1/train_samp_cleaned.csv\")\n",
    "df[\"tgldatang\"] = pd.to_datetime(df[\"tgldatang\"], format=\"%Y/%m/%dT%H:%M:%S\")\n",
    "df[\"tglpulang\"] = pd.to_datetime(df[\"tglpulang\"], format=\"%Y/%m/%dT%H:%M:%S\")\n",
    "df[\"nday\"] = df[\"tglpulang\"] - df[\"tgldatang\"]\n",
    "df[\"nday\"] = pd.to_timedelta(df[\"nday\"]).dt.days\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>label</th>\n",
       "      <th>nday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  usia jenkel  jenispel       biaya  \\\n",
       "0           165666         486    48      P         2    184300.0   \n",
       "1          1010828         520    63      L         1  10628400.0   \n",
       "2           166042         523    53      P         2    187300.0   \n",
       "3           168937         549    54      P         2    187300.0   \n",
       "4          1005899         549    53      P         2    381600.0   \n",
       "...            ...         ...   ...    ...       ...         ...   \n",
       "11291759   9983563     8527704     0      P         1   3645100.0   \n",
       "11291760  11053870     8527709     0      P         1  12273200.0   \n",
       "11291761   7461049     8527751     0      L         1   4099000.0   \n",
       "11291762   1075162     8527853     0      L         1   3992400.0   \n",
       "11291763    102794     8527876     0      P         1   1624900.0   \n",
       "\n",
       "          jenispulang  label  nday  \n",
       "0                 1.0      0     0  \n",
       "1                 1.0      0     3  \n",
       "2                 1.0      0     0  \n",
       "3                 1.0      0     0  \n",
       "4                 1.0      0     0  \n",
       "...               ...    ...   ...  \n",
       "11291759          1.0      0     4  \n",
       "11291760          1.0      0     4  \n",
       "11291761          1.0      0     7  \n",
       "11291762          1.0      0     2  \n",
       "11291763          1.0      0     2  \n",
       "\n",
       "[11291764 rows x 9 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# droping tglpulang tgldatang\n",
    "df.drop([\"tgldatang\", \"tglpulang\"], axis=1, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save to some folder\n",
    "df.to_csv(\"temp_data/1/ready_train_samp.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handle nproc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ewwwr\\AppData\\Local\\Temp\\ipykernel_5428\\1211103821.py:6: DtypeWarning: Columns (1) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(\"temp_data/1/train_proc_cleaned.csv\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>69.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120158</th>\n",
       "      <td>11426915</td>\n",
       "      <td>93.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120159</th>\n",
       "      <td>11426927</td>\n",
       "      <td>89.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120160</th>\n",
       "      <td>11426927</td>\n",
       "      <td>96.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120161</th>\n",
       "      <td>11426927</td>\n",
       "      <td>87.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120162</th>\n",
       "      <td>11426927</td>\n",
       "      <td>93.96</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12120163 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id   proc\n",
       "0                6  90.59\n",
       "1                6  69.01\n",
       "2                6  88.78\n",
       "3               27  90.59\n",
       "4               27  88.78\n",
       "...            ...    ...\n",
       "12120158  11426915  93.57\n",
       "12120159  11426927  89.52\n",
       "12120160  11426927  96.55\n",
       "12120161  11426927  87.39\n",
       "12120162  11426927  93.96\n",
       "\n",
       "[12120163 rows x 2 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the procedure dataset is counted how many procedure each id have and save in nproc column\n",
    "\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "df = pd.read_csv(\"temp_data/1/train_proc_cleaned.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>90.59</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>69.01</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>88.78</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>90.59</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27</td>\n",
       "      <td>88.78</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120158</th>\n",
       "      <td>11426915</td>\n",
       "      <td>93.57</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120159</th>\n",
       "      <td>11426927</td>\n",
       "      <td>89.52</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120160</th>\n",
       "      <td>11426927</td>\n",
       "      <td>96.55</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120161</th>\n",
       "      <td>11426927</td>\n",
       "      <td>87.39</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120162</th>\n",
       "      <td>11426927</td>\n",
       "      <td>93.96</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12120163 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id   proc  nproc\n",
       "0                6  90.59    3.0\n",
       "1                6  69.01    3.0\n",
       "2                6  88.78    3.0\n",
       "3               27  90.59    2.0\n",
       "4               27  88.78    2.0\n",
       "...            ...    ...    ...\n",
       "12120158  11426915  93.57    1.0\n",
       "12120159  11426927  89.52    4.0\n",
       "12120160  11426927  96.55    4.0\n",
       "12120161  11426927  87.39    4.0\n",
       "12120162  11426927  93.96    4.0\n",
       "\n",
       "[12120163 rows x 3 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# counting the duplicate id for nproc\n",
    "# same as in the eda phase\n",
    "\n",
    "temp = dict(df[df[\"id\"].duplicated(keep=False)][\"id\"].value_counts())\n",
    "temp = pd.DataFrame({\n",
    "    \"id\":list(temp.keys()),\n",
    "    \"nproc\":list(temp.values())\n",
    "})\n",
    "df = pd.merge(df, temp, on=\"id\", how=\"left\")\n",
    "df[\"nproc\"].fillna(1, inplace=True)\n",
    "del temp\n",
    "gc.collect()\n",
    "# print(1)\n",
    "\n",
    "# # one hot encoding\n",
    "# encoded_proc = pd.get_dummies(df[\"proc\"])\n",
    "# encoded_proc.drop([2424], axis=1, inplace=True)\n",
    "# df.drop([\"proc\"], axis=1, inplace=True)\n",
    "# print(2)\n",
    "\n",
    "# # just in case if there is incomplete procedure in dataset eg only 0-50 not 0-99\n",
    "# # so im creating mechanism that if thereis only 0-50 im creating the 51-99 column\n",
    "# ls_proc = [x for x in range(100)]\n",
    "# ls_columns = list(encoded_proc.columns)\n",
    "# temp = {}\n",
    "# for i in ls_proc:\n",
    "#     if i not in ls_columns:\n",
    "#         temp.__setitem__(i, [None])\n",
    "# print(3)\n",
    "\n",
    "# # concat the created column to encoded procedure\n",
    "# temp = pd.DataFrame(temp)\n",
    "# encoded_proc = pd.concat([encoded_proc, temp], axis=1)\n",
    "# encoded_proc.sort_index(axis=1, inplace=True)\n",
    "# encoded_proc.fillna(0, inplace=True)\n",
    "# print(4)\n",
    "\n",
    "# # concat encoded procedure to dataframe\n",
    "# df = pd.concat([df, encoded_proc], axis=1)\n",
    "# del encoded_proc, temp\n",
    "# gc.collect()\n",
    "# print(5)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id       0\n",
       "proc     0\n",
       "nproc    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check null\n",
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>29</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>30</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>33</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120155</th>\n",
       "      <td>11426788</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120156</th>\n",
       "      <td>11426822</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120157</th>\n",
       "      <td>11426839</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120158</th>\n",
       "      <td>11426915</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120159</th>\n",
       "      <td>11426927</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6372051 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  nproc\n",
       "0                6    3.0\n",
       "3               27    2.0\n",
       "5               29    1.0\n",
       "6               30    1.0\n",
       "7               33    2.0\n",
       "...            ...    ...\n",
       "12120155  11426788    1.0\n",
       "12120156  11426822    1.0\n",
       "12120157  11426839    1.0\n",
       "12120158  11426915    1.0\n",
       "12120159  11426927    4.0\n",
       "\n",
       "[6372051 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop proc column and drop duplicate\n",
    "df.drop(\"proc\", axis=1, inplace=True)\n",
    "df.drop_duplicates(subset=\"id\", inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>29</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>30</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>33</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120155</th>\n",
       "      <td>11426788</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120156</th>\n",
       "      <td>11426822</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120157</th>\n",
       "      <td>11426839</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120158</th>\n",
       "      <td>11426915</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12120159</th>\n",
       "      <td>11426927</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6372051 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  nproc\n",
       "0                6    3.0\n",
       "3               27    2.0\n",
       "5               29    1.0\n",
       "6               30    1.0\n",
       "7               33    2.0\n",
       "...            ...    ...\n",
       "12120155  11426788    1.0\n",
       "12120156  11426822    1.0\n",
       "12120157  11426839    1.0\n",
       "12120158  11426915    1.0\n",
       "12120159  11426927    4.0\n",
       "\n",
       "[6372051 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save to some folder\n",
    "df.to_csv(\"temp_data/1/ready_train_proc.csv\", index=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handle nsdiag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>diag</th>\n",
       "      <th>levelid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>111</td>\n",
       "      <td>O86.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>133</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>231</td>\n",
       "      <td>E11.9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>231</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>248</td>\n",
       "      <td>O82.1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905250</th>\n",
       "      <td>11426932</td>\n",
       "      <td>M79.1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905251</th>\n",
       "      <td>11426946</td>\n",
       "      <td>N30.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905252</th>\n",
       "      <td>11426947</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905253</th>\n",
       "      <td>11426947</td>\n",
       "      <td>E78.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905254</th>\n",
       "      <td>11426952</td>\n",
       "      <td>M54.5</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5905255 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id   diag  levelid\n",
       "0             111  O86.0        2\n",
       "1             133    K30        2\n",
       "2             231  E11.9        2\n",
       "3             231    K30        2\n",
       "4             248  O82.1        2\n",
       "...           ...    ...      ...\n",
       "5905250  11426932  M79.1        2\n",
       "5905251  11426946  N30.0        2\n",
       "5905252  11426947    K30        2\n",
       "5905253  11426947  E78.0        2\n",
       "5905254  11426952  M54.5        2\n",
       "\n",
       "[5905255 rows x 3 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading data\n",
    "\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "df = pd.read_csv(\"temp_data/1/train_diag_cleaned.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>diag</th>\n",
       "      <th>levelid</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>111</td>\n",
       "      <td>O86.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>133</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>231</td>\n",
       "      <td>E11.9</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>231</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>248</td>\n",
       "      <td>O82.1</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905250</th>\n",
       "      <td>11426932</td>\n",
       "      <td>M79.1</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905251</th>\n",
       "      <td>11426946</td>\n",
       "      <td>N30.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905252</th>\n",
       "      <td>11426947</td>\n",
       "      <td>K30</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905253</th>\n",
       "      <td>11426947</td>\n",
       "      <td>E78.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905254</th>\n",
       "      <td>11426952</td>\n",
       "      <td>M54.5</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5905255 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id   diag  levelid  nsdiag\n",
       "0             111  O86.0        2     1.0\n",
       "1             133    K30        2     1.0\n",
       "2             231  E11.9        2     2.0\n",
       "3             231    K30        2     2.0\n",
       "4             248  O82.1        2     2.0\n",
       "...           ...    ...      ...     ...\n",
       "5905250  11426932  M79.1        2     1.0\n",
       "5905251  11426946  N30.0        2     1.0\n",
       "5905252  11426947    K30        2     2.0\n",
       "5905253  11426947  E78.0        2     2.0\n",
       "5905254  11426952  M54.5        2     1.0\n",
       "\n",
       "[5905255 rows x 4 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# counting the duplicate id for secondary diagnose as nsdiag value\n",
    "# same as in the eda phase\n",
    "temp = dict(df[df[\"id\"].duplicated(keep=False)][\"id\"].value_counts())\n",
    "temp = pd.DataFrame({\n",
    "    \"id\":list(temp.keys()),\n",
    "    \"nsdiag\":list(temp.values())\n",
    "})\n",
    "df = pd.merge(df, temp, on=\"id\", how=\"left\")\n",
    "df[\"nsdiag\"].fillna(1, inplace=True)\n",
    "del temp\n",
    "gc.collect()\n",
    "# print(1)\n",
    "\n",
    "# # one hot encoding\n",
    "# encoded_proc = pd.get_dummies(df[\"proc\"])\n",
    "# encoded_proc.drop([2424], axis=1, inplace=True)\n",
    "# df.drop([\"proc\"], axis=1, inplace=True)\n",
    "# print(2)\n",
    "\n",
    "# # just in case if there is incomplete procedure in dataset eg only 0-50 not 0-99\n",
    "# # so im creating mechanism that if thereis only 0-50 im creating the 51-99 column\n",
    "# ls_proc = [x for x in range(100)]\n",
    "# ls_columns = list(encoded_proc.columns)\n",
    "# temp = {}\n",
    "# for i in ls_proc:\n",
    "#     if i not in ls_columns:\n",
    "#         temp.__setitem__(i, [None])\n",
    "# print(3)\n",
    "\n",
    "# # concat the created column to encoded procedure\n",
    "# temp = pd.DataFrame(temp)\n",
    "# encoded_proc = pd.concat([encoded_proc, temp], axis=1)\n",
    "# encoded_proc.sort_index(axis=1, inplace=True)\n",
    "# encoded_proc.fillna(0, inplace=True)\n",
    "# print(4)\n",
    "\n",
    "# # concat encoded procedure to dataframe\n",
    "# df = pd.concat([df, encoded_proc], axis=1)\n",
    "# del encoded_proc, temp\n",
    "# gc.collect()\n",
    "# print(5)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id         0\n",
       "diag       0\n",
       "levelid    0\n",
       "nsdiag     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check null\n",
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>111</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>133</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>231</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>248</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>259</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905249</th>\n",
       "      <td>11426929</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905250</th>\n",
       "      <td>11426932</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905251</th>\n",
       "      <td>11426946</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905252</th>\n",
       "      <td>11426947</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905254</th>\n",
       "      <td>11426952</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3815183 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id  nsdiag\n",
       "0             111     1.0\n",
       "1             133     1.0\n",
       "2             231     2.0\n",
       "4             248     2.0\n",
       "6             259     1.0\n",
       "...           ...     ...\n",
       "5905249  11426929     1.0\n",
       "5905250  11426932     1.0\n",
       "5905251  11426946     1.0\n",
       "5905252  11426947     2.0\n",
       "5905254  11426952     1.0\n",
       "\n",
       "[3815183 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop proc column and drop duplicate\n",
    "df.drop([\"diag\", \"levelid\"], axis=1, inplace=True)\n",
    "df.drop_duplicates(subset=\"id\", inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>111</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>133</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>231</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>248</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>259</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905249</th>\n",
       "      <td>11426929</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905250</th>\n",
       "      <td>11426932</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905251</th>\n",
       "      <td>11426946</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905252</th>\n",
       "      <td>11426947</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5905254</th>\n",
       "      <td>11426952</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3815183 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id  nsdiag\n",
       "0             111     1.0\n",
       "1             133     1.0\n",
       "2             231     2.0\n",
       "4             248     2.0\n",
       "6             259     1.0\n",
       "...           ...     ...\n",
       "5905249  11426929     1.0\n",
       "5905250  11426932     1.0\n",
       "5905251  11426946     1.0\n",
       "5905252  11426947     2.0\n",
       "5905254  11426952     1.0\n",
       "\n",
       "[3815183 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save to some folder\n",
    "df.to_csv(\"temp_data/1/ready_train_diag.csv\", index=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoding Categorical Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for categorical data im encode it and one hot enocode it but im only creating a function here\n",
    "# this function called from somewhere soon because for ram reason im not encoding all 11M+ data\n",
    "# only data that il use in train and test to encode\n",
    "\n",
    "# after one hot encoding im dropping one column because its for efficiency\n",
    "# example there is sex data that have value P and L after one hot encoding it will be have P colum and L colum\n",
    "# if male it will be 1 for P and 0 for L, actualy it can be described only one of those column\n",
    "# so if i drop no matter what column eg the L column, if its male the P is 0 and female P is 1\n",
    "\n",
    "from pandas import DataFrame\n",
    "import numpy as np\n",
    "import gc\n",
    "\n",
    "def encode_categorical(df:DataFrame):\n",
    "\n",
    "    try:\n",
    "        df[\"id\"] = df[\"id\"].astype(np.int64)\n",
    "    except:\n",
    "        print(\"no id column skip formating for id column to np.int64\")\n",
    "    try:\n",
    "        df[\"typefaskes\"] = df[\"typefaskes\"].astype(str)\n",
    "    except:\n",
    "        print(\"no typefaskes column skip formating for typefaskes column to str\")\n",
    "    try:\n",
    "        df[\"kelasrawat\"] = df[\"kelasrawat\"].astype(np.int8)\n",
    "    except:\n",
    "        print(\"no kelasrawat column skip formating for kelasrawat column to str\")\n",
    "\n",
    "    df[\"usia\"] = df[\"usia\"].astype(np.int8)\n",
    "    df[\"jenkel\"] = df[\"jenkel\"].astype(str)\n",
    "    df[\"jenispel\"] = df[\"jenispel\"].astype(str)\n",
    "    df[\"jenispulang\"] = df[\"jenispulang\"].astype(str)\n",
    "    df[\"biaya\"] = df[\"biaya\"].astype(float)\n",
    "\n",
    "    try:\n",
    "        df[\"label\"] = df[\"label\"].astype(np.int8)\n",
    "    except:\n",
    "        print(\"no label column skip formating for label column to np.int8\")\n",
    "\n",
    "    try:\n",
    "        encoded_typefaskes = pd.get_dummies(df[\"typefaskes\"])\n",
    "        try:\n",
    "            encoded_typefaskes.drop([\"KK\"], axis=1, inplace=True)\n",
    "        except:\n",
    "            print(\"no KK column skip creating KK column\")\n",
    "        df.drop([\"typefaskes\"], axis=1, inplace=True)\n",
    "        df = pd.concat([df, encoded_typefaskes], axis=1)\n",
    "        del encoded_typefaskes\n",
    "        gc.collect()\n",
    "    except:\n",
    "        print(\"no typefaskes column skip encoding typefaskes\")\n",
    "\n",
    "    encoded_jenkel = pd.get_dummies(df[\"jenkel\"])\n",
    "    encoded_jenkel.drop([\"P\"], axis=1, inplace=True)\n",
    "    encoded_jenkel = encoded_jenkel.rename(columns={\"L\":\"ismale\"})\n",
    "    df.drop([\"jenkel\"], axis=1, inplace=True)\n",
    "    df = pd.concat([df, encoded_jenkel], axis=1)\n",
    "    del encoded_jenkel\n",
    "    gc.collect()\n",
    "\n",
    "    encoded_jenispel = pd.get_dummies(df[\"jenispel\"])\n",
    "    encoded_jenispel.drop([\"2\"], axis=1, inplace=True)\n",
    "    encoded_jenispel = encoded_jenispel.rename(columns={\"1\":\"jenpel1\"})\n",
    "    df.drop([\"jenispel\"], axis=1, inplace=True)\n",
    "    df = pd.concat([df, encoded_jenispel], axis=1)\n",
    "    del encoded_jenispel\n",
    "    gc.collect()\n",
    "\n",
    "    encoded_jenispulang = pd.get_dummies(df[\"jenispulang\"])\n",
    "    encoded_jenispulang.drop([\"5.0\"], axis=1, inplace=True)\n",
    "    encoded_jenispulang = encoded_jenispulang.rename(columns={\n",
    "                                                    \"1.0\":\"jenpul1\",\n",
    "                                                    \"2.0\":\"jenpul2\",\n",
    "                                                    \"3.0\":\"jenpul3\",\n",
    "                                                    \"4.0\":\"jenpul4\",\n",
    "                                                })\n",
    "    df.drop([\"jenispulang\"], axis=1, inplace=True)\n",
    "    df = pd.concat([df, encoded_jenispulang], axis=1)\n",
    "    del encoded_jenispulang\n",
    "    gc.collect()\n",
    "\n",
    "    # encoded_proc = pd.get_dummies(df[\"proc\"])\n",
    "    # encoded_proc.drop([\"2424\"], axis=1, inplace=True)\n",
    "    # df.drop([\"proc\"], axis=1, inplace=True)\n",
    "    # df = pd.concat([df, encoded_proc], axis=1)\n",
    "    # del encoded_proc\n",
    "    # gc.collect()\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Balancing Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mixed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# in here im implementing \n",
    "# undersampling the majority class (label 0) because if i oversampling the 1 class (150k) to 0 class (11M+)\n",
    "# i think its too much, so in the label 0 class il take it 2.000.000 samples\n",
    "# oversampling minority class (label 1) using SMOTE\n",
    "# so i call it mixed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>label</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  usia jenkel  jenispel       biaya  \\\n",
       "0           165666         486    48      P         2    184300.0   \n",
       "1          1010828         520    63      L         1  10628400.0   \n",
       "2           166042         523    53      P         2    187300.0   \n",
       "3           168937         549    54      P         2    187300.0   \n",
       "4          1005899         549    53      P         2    381600.0   \n",
       "...            ...         ...   ...    ...       ...         ...   \n",
       "11291759   9983563     8527704     0      P         1   3645100.0   \n",
       "11291760  11053870     8527709     0      P         1  12273200.0   \n",
       "11291761   7461049     8527751     0      L         1   4099000.0   \n",
       "11291762   1075162     8527853     0      L         1   3992400.0   \n",
       "11291763    102794     8527876     0      P         1   1624900.0   \n",
       "\n",
       "          jenispulang  label  nday  nproc  \n",
       "0                 1.0      0     0    NaN  \n",
       "1                 1.0      0     3    1.0  \n",
       "2                 1.0      0     0    NaN  \n",
       "3                 1.0      0     0    NaN  \n",
       "4                 1.0      0     0    NaN  \n",
       "...               ...    ...   ...    ...  \n",
       "11291759          1.0      0     4    4.0  \n",
       "11291760          1.0      0     4    5.0  \n",
       "11291761          1.0      0     7    1.0  \n",
       "11291762          1.0      0     2    1.0  \n",
       "11291763          1.0      0     2    NaN  \n",
       "\n",
       "[11291764 rows x 10 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading the data and join using left join\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"temp_data/1/ready_train_samp.csv\")\n",
    "proc = pd.read_csv(\"temp_data/1/ready_train_proc.csv\")\n",
    "diag = pd.read_csv(\"temp_data/1/ready_train_diag.csv\")\n",
    "\n",
    "df = pd.merge(df, proc, on=\"id\", how=\"left\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>label</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  usia jenkel  jenispel       biaya  \\\n",
       "0           165666         486    48      P         2    184300.0   \n",
       "1          1010828         520    63      L         1  10628400.0   \n",
       "2           166042         523    53      P         2    187300.0   \n",
       "3           168937         549    54      P         2    187300.0   \n",
       "4          1005899         549    53      P         2    381600.0   \n",
       "...            ...         ...   ...    ...       ...         ...   \n",
       "11291759   9983563     8527704     0      P         1   3645100.0   \n",
       "11291760  11053870     8527709     0      P         1  12273200.0   \n",
       "11291761   7461049     8527751     0      L         1   4099000.0   \n",
       "11291762   1075162     8527853     0      L         1   3992400.0   \n",
       "11291763    102794     8527876     0      P         1   1624900.0   \n",
       "\n",
       "          jenispulang  label  nday  nproc  nsdiag  \n",
       "0                 1.0      0     0    NaN     NaN  \n",
       "1                 1.0      0     3    1.0     1.0  \n",
       "2                 1.0      0     0    NaN     2.0  \n",
       "3                 1.0      0     0    NaN     2.0  \n",
       "4                 1.0      0     0    NaN     NaN  \n",
       "...               ...    ...   ...    ...     ...  \n",
       "11291759          1.0      0     4    4.0     1.0  \n",
       "11291760          1.0      0     4    5.0     1.0  \n",
       "11291761          1.0      0     7    1.0     1.0  \n",
       "11291762          1.0      0     2    1.0     NaN  \n",
       "11291763          1.0      0     2    NaN     NaN  \n",
       "\n",
       "[11291764 rows x 11 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# left join the nsdiag column\n",
    "df = pd.merge(df, diag, on=\"id\", how=\"left\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "id_peserta     0\n",
       "usia           0\n",
       "jenkel         0\n",
       "jenispel       0\n",
       "biaya          0\n",
       "jenispulang    0\n",
       "label          0\n",
       "nday           0\n",
       "nproc          0\n",
       "nsdiag         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fill nproc null value with 0 because if null there is no procedure and nsdiag counted in proc diag table\n",
    "df[\"nproc\"].fillna(0, inplace=True)\n",
    "df[\"nsdiag\"].fillna(0, inplace=True)\n",
    "df.isna().sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    11136086\n",
       "1      155678\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the imbalance\n",
    "df[\"label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>label</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>486</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>520</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>523</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>549</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>549</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>9983563</td>\n",
       "      <td>8527704</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>11053870</td>\n",
       "      <td>8527709</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>7461049</td>\n",
       "      <td>8527751</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>1075162</td>\n",
       "      <td>8527853</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>102794</td>\n",
       "      <td>8527876</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  id_peserta  usia jenkel  jenispel       biaya  \\\n",
       "0           165666         486    48      P         2    184300.0   \n",
       "1          1010828         520    63      L         1  10628400.0   \n",
       "2           166042         523    53      P         2    187300.0   \n",
       "3           168937         549    54      P         2    187300.0   \n",
       "4          1005899         549    53      P         2    381600.0   \n",
       "...            ...         ...   ...    ...       ...         ...   \n",
       "11291759   9983563     8527704     0      P         1   3645100.0   \n",
       "11291760  11053870     8527709     0      P         1  12273200.0   \n",
       "11291761   7461049     8527751     0      L         1   4099000.0   \n",
       "11291762   1075162     8527853     0      L         1   3992400.0   \n",
       "11291763    102794     8527876     0      P         1   1624900.0   \n",
       "\n",
       "          jenispulang  label  nday  nproc  nsdiag  \n",
       "0                 1.0      0     0    0.0     0.0  \n",
       "1                 1.0      0     3    1.0     1.0  \n",
       "2                 1.0      0     0    0.0     2.0  \n",
       "3                 1.0      0     0    0.0     2.0  \n",
       "4                 1.0      0     0    0.0     0.0  \n",
       "...               ...    ...   ...    ...     ...  \n",
       "11291759          1.0      0     4    4.0     1.0  \n",
       "11291760          1.0      0     4    5.0     1.0  \n",
       "11291761          1.0      0     7    1.0     1.0  \n",
       "11291762          1.0      0     2    1.0     0.0  \n",
       "11291763          1.0      0     2    0.0     0.0  \n",
       "\n",
       "[11291764 rows x 11 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# view the data\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>label</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>165666</td>\n",
       "      <td>48</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>184300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1010828</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>10628400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>166042</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168937</td>\n",
       "      <td>54</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1005899</td>\n",
       "      <td>53</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291759</th>\n",
       "      <td>9983563</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>3645100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291760</th>\n",
       "      <td>11053870</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>12273200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291761</th>\n",
       "      <td>7461049</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>4099000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291762</th>\n",
       "      <td>1075162</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>3992400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11291763</th>\n",
       "      <td>102794</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>1624900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11291764 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  usia jenkel  jenispel       biaya  jenispulang  label  \\\n",
       "0           165666    48      P         2    184300.0          1.0      0   \n",
       "1          1010828    63      L         1  10628400.0          1.0      0   \n",
       "2           166042    53      P         2    187300.0          1.0      0   \n",
       "3           168937    54      P         2    187300.0          1.0      0   \n",
       "4          1005899    53      P         2    381600.0          1.0      0   \n",
       "...            ...   ...    ...       ...         ...          ...    ...   \n",
       "11291759   9983563     0      P         1   3645100.0          1.0      0   \n",
       "11291760  11053870     0      P         1  12273200.0          1.0      0   \n",
       "11291761   7461049     0      L         1   4099000.0          1.0      0   \n",
       "11291762   1075162     0      L         1   3992400.0          1.0      0   \n",
       "11291763    102794     0      P         1   1624900.0          1.0      0   \n",
       "\n",
       "          nday  nproc  nsdiag  \n",
       "0            0    0.0     0.0  \n",
       "1            3    1.0     1.0  \n",
       "2            0    0.0     2.0  \n",
       "3            0    0.0     2.0  \n",
       "4            0    0.0     0.0  \n",
       "...        ...    ...     ...  \n",
       "11291759     4    4.0     1.0  \n",
       "11291760     4    5.0     1.0  \n",
       "11291761     7    1.0     1.0  \n",
       "11291762     2    1.0     0.0  \n",
       "11291763     2    0.0     0.0  \n",
       "\n",
       "[11291764 rows x 10 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop id_peserta\n",
    "df.drop(\"id_peserta\", axis=1, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x for all column except label\n",
    "# y for label column only\n",
    "\n",
    "x = df.drop([\"label\"], axis=1)\n",
    "y = df[\"label\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    8909038\n",
      "1     124373\n",
      "Name: label, dtype: int64\n",
      "0    2227048\n",
      "1      31305\n",
      "Name: label, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# before balancing the data i split the train data into train and test\n",
    "# because i dont want the balanced/artificial data is used during testing\n",
    "\n",
    "# split the data into train and test\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.2, random_state=24)\n",
    "print(ytrain.value_counts())\n",
    "print(ytest.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6614422</th>\n",
       "      <td>9360097</td>\n",
       "      <td>44</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>7085700.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4712123</th>\n",
       "      <td>9471956</td>\n",
       "      <td>51</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>215000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7679502</th>\n",
       "      <td>2980934</td>\n",
       "      <td>65</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>185000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8982509</th>\n",
       "      <td>5023565</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>115800.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3416670</th>\n",
       "      <td>116369</td>\n",
       "      <td>41</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>130400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5657729</th>\n",
       "      <td>6983324</td>\n",
       "      <td>3</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>184600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4647057</th>\n",
       "      <td>936902</td>\n",
       "      <td>67</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>190400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8905047</th>\n",
       "      <td>593607</td>\n",
       "      <td>17</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1259712</th>\n",
       "      <td>7842603</td>\n",
       "      <td>49</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>190400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6292355</th>\n",
       "      <td>10709474</td>\n",
       "      <td>54</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>1257500.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9033411 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id  usia jenkel  jenispel      biaya  jenispulang  nday  nproc  \\\n",
       "6614422   9360097    44      L         1  7085700.0          1.0     7    5.0   \n",
       "4712123   9471956    51      P         2   215000.0          1.0     0    2.0   \n",
       "7679502   2980934    65      L         2   185000.0          1.0     0    0.0   \n",
       "8982509   5023565    66      P         2   115800.0          1.0     0    3.0   \n",
       "3416670    116369    41      L         2   130400.0          1.0     0    0.0   \n",
       "...           ...   ...    ...       ...        ...          ...   ...    ...   \n",
       "5657729   6983324     3      L         2   184600.0          1.0     0    0.0   \n",
       "4647057    936902    67      P         2   190400.0          1.0     0    0.0   \n",
       "8905047    593607    17      P         2        0.0          1.0     0    1.0   \n",
       "1259712   7842603    49      P         2   190400.0          1.0     0    1.0   \n",
       "6292355  10709474    54      L         1  1257500.0          1.0     2    1.0   \n",
       "\n",
       "         nsdiag  label  \n",
       "6614422     5.0      0  \n",
       "4712123     0.0      0  \n",
       "7679502     0.0      0  \n",
       "8982509     1.0      0  \n",
       "3416670     0.0      0  \n",
       "...         ...    ...  \n",
       "5657729     0.0      0  \n",
       "4647057     1.0      0  \n",
       "8905047     0.0      1  \n",
       "1259712     1.0      0  \n",
       "6292355     0.0      0  \n",
       "\n",
       "[9033411 rows x 10 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# before performing sampling the train data im merging x and y for train\n",
    "# because if not when i performing sampling the index of the record data for x and y will not match\n",
    "\n",
    "xtrain.insert(len(xtrain.columns), \"label\", ytrain)\n",
    "xtrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6614422</th>\n",
       "      <td>9360097</td>\n",
       "      <td>44</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>7085700.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4712123</th>\n",
       "      <td>9471956</td>\n",
       "      <td>51</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>215000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7679502</th>\n",
       "      <td>2980934</td>\n",
       "      <td>65</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>185000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8982509</th>\n",
       "      <td>5023565</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>115800.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3416670</th>\n",
       "      <td>116369</td>\n",
       "      <td>41</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>130400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7221371</th>\n",
       "      <td>8860902</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>483400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5657729</th>\n",
       "      <td>6983324</td>\n",
       "      <td>3</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>184600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4647057</th>\n",
       "      <td>936902</td>\n",
       "      <td>67</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>190400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1259712</th>\n",
       "      <td>7842603</td>\n",
       "      <td>49</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>190400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6292355</th>\n",
       "      <td>10709474</td>\n",
       "      <td>54</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>1257500.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8909038 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id  usia jenkel  jenispel      biaya  jenispulang  nday  nproc  \\\n",
       "6614422   9360097    44      L         1  7085700.0          1.0     7    5.0   \n",
       "4712123   9471956    51      P         2   215000.0          1.0     0    2.0   \n",
       "7679502   2980934    65      L         2   185000.0          1.0     0    0.0   \n",
       "8982509   5023565    66      P         2   115800.0          1.0     0    3.0   \n",
       "3416670    116369    41      L         2   130400.0          1.0     0    0.0   \n",
       "...           ...   ...    ...       ...        ...          ...   ...    ...   \n",
       "7221371   8860902     0      L         2   483400.0          1.0     0    2.0   \n",
       "5657729   6983324     3      L         2   184600.0          1.0     0    0.0   \n",
       "4647057    936902    67      P         2   190400.0          1.0     0    0.0   \n",
       "1259712   7842603    49      P         2   190400.0          1.0     0    1.0   \n",
       "6292355  10709474    54      L         1  1257500.0          1.0     2    1.0   \n",
       "\n",
       "         nsdiag  label  \n",
       "6614422     5.0      0  \n",
       "4712123     0.0      0  \n",
       "7679502     0.0      0  \n",
       "8982509     1.0      0  \n",
       "3416670     0.0      0  \n",
       "...         ...    ...  \n",
       "7221371     0.0      0  \n",
       "5657729     0.0      0  \n",
       "4647057     1.0      0  \n",
       "1259712     1.0      0  \n",
       "6292355     0.0      0  \n",
       "\n",
       "[8909038 rows x 10 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set the all 0 label to train_0 for soon performing undersampling for this 0 class\n",
    "train_0 = xtrain[xtrain[\"label\"] == 0]\n",
    "train_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8560792</th>\n",
       "      <td>8832961</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5358456</th>\n",
       "      <td>2285153</td>\n",
       "      <td>35</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>6965200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3400468</th>\n",
       "      <td>6131708</td>\n",
       "      <td>8</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>380500.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10912280</th>\n",
       "      <td>2017795</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3432450</th>\n",
       "      <td>5776482</td>\n",
       "      <td>1</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>175000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9950303</th>\n",
       "      <td>10259923</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2996087</th>\n",
       "      <td>8519005</td>\n",
       "      <td>43</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9650805</th>\n",
       "      <td>5240470</td>\n",
       "      <td>55</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>213100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11244970</th>\n",
       "      <td>9741448</td>\n",
       "      <td>62</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8905047</th>\n",
       "      <td>593607</td>\n",
       "      <td>17</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>124373 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  usia jenkel  jenispel      biaya  jenispulang  nday  \\\n",
       "8560792    8832961    63      L         2        0.0          1.0     0   \n",
       "5358456    2285153    35      P         1  6965200.0          1.0     3   \n",
       "3400468    6131708     8      P         2   380500.0          1.0     0   \n",
       "10912280   2017795    66      P         2        0.0          1.0     0   \n",
       "3432450    5776482     1      P         2   175000.0          1.0     0   \n",
       "...            ...   ...    ...       ...        ...          ...   ...   \n",
       "9950303   10259923    63      L         2        0.0          1.0     0   \n",
       "2996087    8519005    43      L         2        0.0          1.0     0   \n",
       "9650805    5240470    55      P         2   213100.0          1.0     0   \n",
       "11244970   9741448    62      L         2        0.0          1.0     0   \n",
       "8905047     593607    17      P         2        0.0          1.0     0   \n",
       "\n",
       "          nproc  nsdiag  label  \n",
       "8560792     0.0     0.0      1  \n",
       "5358456     1.0     3.0      1  \n",
       "3400468     1.0     0.0      1  \n",
       "10912280    1.0     0.0      1  \n",
       "3432450     0.0     0.0      1  \n",
       "...         ...     ...    ...  \n",
       "9950303     1.0     0.0      1  \n",
       "2996087     2.0     1.0      1  \n",
       "9650805     1.0     2.0      1  \n",
       "11244970    4.0     0.0      1  \n",
       "8905047     1.0     0.0      1  \n",
       "\n",
       "[124373 rows x 10 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# keep the 1 class to train_1 class soon it will merge with undersampled 0 class\n",
    "train_1 = xtrain[xtrain[\"label\"] == 1]\n",
    "train_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7570239</td>\n",
       "      <td>9</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>967000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4979730</td>\n",
       "      <td>31</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>190000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8362635</td>\n",
       "      <td>25</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>9989100.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1028829</td>\n",
       "      <td>50</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>4485000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10427508</td>\n",
       "      <td>1</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>2025500.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124368</th>\n",
       "      <td>10259923</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124369</th>\n",
       "      <td>8519005</td>\n",
       "      <td>43</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124370</th>\n",
       "      <td>5240470</td>\n",
       "      <td>55</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>213100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124371</th>\n",
       "      <td>9741448</td>\n",
       "      <td>62</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124372</th>\n",
       "      <td>593607</td>\n",
       "      <td>17</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2124373 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id  usia jenkel  jenispel      biaya  jenispulang  nday  nproc  \\\n",
       "0         7570239     9      L         2   967000.0          1.0     0    1.0   \n",
       "1         4979730    31      P         2   190000.0          1.0     0    0.0   \n",
       "2         8362635    25      P         1  9989100.0          4.0     4    4.0   \n",
       "3         1028829    50      L         1  4485000.0          1.0     6    2.0   \n",
       "4        10427508     1      P         1  2025500.0          1.0     1    0.0   \n",
       "...           ...   ...    ...       ...        ...          ...   ...    ...   \n",
       "2124368  10259923    63      L         2        0.0          1.0     0    1.0   \n",
       "2124369   8519005    43      L         2        0.0          1.0     0    2.0   \n",
       "2124370   5240470    55      P         2   213100.0          1.0     0    1.0   \n",
       "2124371   9741448    62      L         2        0.0          1.0     0    4.0   \n",
       "2124372    593607    17      P         2        0.0          1.0     0    1.0   \n",
       "\n",
       "         nsdiag  label  \n",
       "0           0.0      0  \n",
       "1           0.0      0  \n",
       "2           2.0      0  \n",
       "3           1.0      0  \n",
       "4           0.0      0  \n",
       "...         ...    ...  \n",
       "2124368     0.0      1  \n",
       "2124369     1.0      1  \n",
       "2124370     2.0      1  \n",
       "2124371     0.0      1  \n",
       "2124372     0.0      1  \n",
       "\n",
       "[2124373 rows x 10 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# undersampling the train0 data into 2.000.000\n",
    "# after undersampled im appending the 1 class data below of the 0 class data\n",
    "train = train_0.sample(n=2_000_000, random_state=24)\n",
    "train = pd.concat([train, train_1], ignore_index=True)\n",
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no typefaskes column skip formating for typefaskes column to str\n",
      "no kelasrawat column skip formating for kelasrawat column to str\n",
      "no typefaskes column skip encoding typefaskes\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>biaya</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>label</th>\n",
       "      <th>ismale</th>\n",
       "      <th>jenpel1</th>\n",
       "      <th>jenpul1</th>\n",
       "      <th>jenpul2</th>\n",
       "      <th>jenpul3</th>\n",
       "      <th>jenpul4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7570239</td>\n",
       "      <td>9</td>\n",
       "      <td>967000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4979730</td>\n",
       "      <td>31</td>\n",
       "      <td>190000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8362635</td>\n",
       "      <td>25</td>\n",
       "      <td>9989100.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1028829</td>\n",
       "      <td>50</td>\n",
       "      <td>4485000.0</td>\n",
       "      <td>6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10427508</td>\n",
       "      <td>1</td>\n",
       "      <td>2025500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124368</th>\n",
       "      <td>10259923</td>\n",
       "      <td>63</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124369</th>\n",
       "      <td>8519005</td>\n",
       "      <td>43</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124370</th>\n",
       "      <td>5240470</td>\n",
       "      <td>55</td>\n",
       "      <td>213100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124371</th>\n",
       "      <td>9741448</td>\n",
       "      <td>62</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124372</th>\n",
       "      <td>593607</td>\n",
       "      <td>17</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2124373 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id  usia      biaya  nday  nproc  nsdiag  label  ismale  \\\n",
       "0         7570239     9   967000.0     0    1.0     0.0      0       1   \n",
       "1         4979730    31   190000.0     0    0.0     0.0      0       0   \n",
       "2         8362635    25  9989100.0     4    4.0     2.0      0       0   \n",
       "3         1028829    50  4485000.0     6    2.0     1.0      0       1   \n",
       "4        10427508     1  2025500.0     1    0.0     0.0      0       0   \n",
       "...           ...   ...        ...   ...    ...     ...    ...     ...   \n",
       "2124368  10259923    63        0.0     0    1.0     0.0      1       1   \n",
       "2124369   8519005    43        0.0     0    2.0     1.0      1       1   \n",
       "2124370   5240470    55   213100.0     0    1.0     2.0      1       0   \n",
       "2124371   9741448    62        0.0     0    4.0     0.0      1       1   \n",
       "2124372    593607    17        0.0     0    1.0     0.0      1       0   \n",
       "\n",
       "         jenpel1  jenpul1  jenpul2  jenpul3  jenpul4  \n",
       "0              0        1        0        0        0  \n",
       "1              0        1        0        0        0  \n",
       "2              1        0        0        0        1  \n",
       "3              1        1        0        0        0  \n",
       "4              1        1        0        0        0  \n",
       "...          ...      ...      ...      ...      ...  \n",
       "2124368        0        1        0        0        0  \n",
       "2124369        0        1        0        0        0  \n",
       "2124370        0        1        0        0        0  \n",
       "2124371        0        1        0        0        0  \n",
       "2124372        0        1        0        0        0  \n",
       "\n",
       "[2124373 rows x 13 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# here im calling the encode method i have created before\n",
    "# basicaly one hot encoding and dropping one of the column\n",
    "train = encode_categorical(train)\n",
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop id column\n",
    "xtrain.drop(\"id\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2000000\n",
       "1     124373\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# setting again the xtrain and ytrain from undersampled data\n",
    "xtrain = train.drop([\"label\"], axis=1)\n",
    "ytrain = train[\"label\"]\n",
    "ytrain.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# delete var for ram reason\n",
    "del train, train_0, train_1\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2000000\n",
       "1    2000000\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# oversampling the 1 class data using smote from imblearn\n",
    "# and place it in x, y so x and y is the artificial train data\n",
    "from imblearn.over_sampling import SMOTE\n",
    "sampler = SMOTE(sampling_strategy=\"minority\")\n",
    "\n",
    "x, y = sampler.fit_resample(xtrain, ytrain)\n",
    "y.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>biaya</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>ismale</th>\n",
       "      <th>jenpel1</th>\n",
       "      <th>jenpul1</th>\n",
       "      <th>jenpul2</th>\n",
       "      <th>jenpul3</th>\n",
       "      <th>jenpul4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7570239</td>\n",
       "      <td>9</td>\n",
       "      <td>9.670000e+05</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4979730</td>\n",
       "      <td>31</td>\n",
       "      <td>1.900000e+05</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8362635</td>\n",
       "      <td>25</td>\n",
       "      <td>9.989100e+06</td>\n",
       "      <td>4</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1028829</td>\n",
       "      <td>50</td>\n",
       "      <td>4.485000e+06</td>\n",
       "      <td>6</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10427508</td>\n",
       "      <td>1</td>\n",
       "      <td>2.025500e+06</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999995</th>\n",
       "      <td>2305873</td>\n",
       "      <td>50</td>\n",
       "      <td>5.471000e+05</td>\n",
       "      <td>0</td>\n",
       "      <td>0.619181</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999996</th>\n",
       "      <td>4267060</td>\n",
       "      <td>0</td>\n",
       "      <td>7.193563e+06</td>\n",
       "      <td>1</td>\n",
       "      <td>0.280025</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999997</th>\n",
       "      <td>2125682</td>\n",
       "      <td>25</td>\n",
       "      <td>3.821318e+06</td>\n",
       "      <td>1</td>\n",
       "      <td>0.721856</td>\n",
       "      <td>0.556287</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999998</th>\n",
       "      <td>9915976</td>\n",
       "      <td>24</td>\n",
       "      <td>6.801200e+06</td>\n",
       "      <td>3</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999999</th>\n",
       "      <td>6917022</td>\n",
       "      <td>32</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.538381</td>\n",
       "      <td>0.461619</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4000000 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id  usia         biaya  nday     nproc    nsdiag  ismale  \\\n",
       "0         7570239     9  9.670000e+05     0  1.000000  0.000000       1   \n",
       "1         4979730    31  1.900000e+05     0  0.000000  0.000000       0   \n",
       "2         8362635    25  9.989100e+06     4  4.000000  2.000000       0   \n",
       "3         1028829    50  4.485000e+06     6  2.000000  1.000000       1   \n",
       "4        10427508     1  2.025500e+06     1  0.000000  0.000000       0   \n",
       "...           ...   ...           ...   ...       ...       ...     ...   \n",
       "3999995   2305873    50  5.471000e+05     0  0.619181  1.000000       0   \n",
       "3999996   4267060     0  7.193563e+06     1  0.280025  1.000000       0   \n",
       "3999997   2125682    25  3.821318e+06     1  0.721856  0.556287       0   \n",
       "3999998   9915976    24  6.801200e+06     3  2.000000  1.000000       0   \n",
       "3999999   6917022    32  0.000000e+00     0  0.538381  0.461619       0   \n",
       "\n",
       "         jenpel1  jenpul1  jenpul2  jenpul3  jenpul4  \n",
       "0              0        1        0        0        0  \n",
       "1              0        1        0        0        0  \n",
       "2              1        0        0        0        1  \n",
       "3              1        1        0        0        0  \n",
       "4              1        1        0        0        0  \n",
       "...          ...      ...      ...      ...      ...  \n",
       "3999995        0        1        0        0        0  \n",
       "3999996        1        0        0        0        0  \n",
       "3999997        1        1        0        0        0  \n",
       "3999998        1        1        0        0        0  \n",
       "3999999        0        1        0        0        0  \n",
       "\n",
       "[4000000 rows x 12 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check x\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>usia</th>\n",
       "      <th>biaya</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>ismale</th>\n",
       "      <th>jenpel1</th>\n",
       "      <th>jenpul1</th>\n",
       "      <th>jenpul2</th>\n",
       "      <th>jenpul3</th>\n",
       "      <th>jenpul4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>967000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>967000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>31</td>\n",
       "      <td>190000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>190000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>25</td>\n",
       "      <td>9989100</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>9989100</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>50</td>\n",
       "      <td>4485000</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>4485000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2025500</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2025500</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999995</th>\n",
       "      <td>50</td>\n",
       "      <td>547100</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>547100</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999996</th>\n",
       "      <td>0</td>\n",
       "      <td>7193562</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7193562</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999997</th>\n",
       "      <td>25</td>\n",
       "      <td>3821317</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3821317</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999998</th>\n",
       "      <td>24</td>\n",
       "      <td>6801200</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>6801200</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999999</th>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4000000 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         usia    biaya  nday  nproc   nsdiag  ismale  jenpel1  jenpul1  \\\n",
       "0           9   967000     0      1   967000       1        0        1   \n",
       "1          31   190000     0      0   190000       0        0        1   \n",
       "2          25  9989100     4      4  9989100       0        1        0   \n",
       "3          50  4485000     6      2  4485000       1        1        1   \n",
       "4           1  2025500     1      0  2025500       0        1        1   \n",
       "...       ...      ...   ...    ...      ...     ...      ...      ...   \n",
       "3999995    50   547100     0      0   547100       0        0        1   \n",
       "3999996     0  7193562     1      0  7193562       0        1        0   \n",
       "3999997    25  3821317     1      0  3821317       0        1        1   \n",
       "3999998    24  6801200     3      2  6801200       0        1        1   \n",
       "3999999    32        0     0      0        0       0        0        1   \n",
       "\n",
       "         jenpul2  jenpul3  jenpul4  \n",
       "0              0        0        0  \n",
       "1              0        0        0  \n",
       "2              0        0        1  \n",
       "3              0        0        0  \n",
       "4              0        0        0  \n",
       "...          ...      ...      ...  \n",
       "3999995        0        0        0  \n",
       "3999996        0        0        0  \n",
       "3999997        0        0        0  \n",
       "3999998        0        0        0  \n",
       "3999999        0        0        0  \n",
       "\n",
       "[4000000 rows x 11 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# formating variable\n",
    "x[\"nproc\"] = x[\"nproc\"].astype(int)\n",
    "x[\"biaya\"] = x[\"biaya\"].astype(int)\n",
    "x[\"nsdiag\"] = x[\"biaya\"].astype(int)\n",
    "x.drop(\"id\", axis=1, inplace=True)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>usia</th>\n",
       "      <th>biaya</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>ismale</th>\n",
       "      <th>jenpel1</th>\n",
       "      <th>jenpul1</th>\n",
       "      <th>jenpul2</th>\n",
       "      <th>jenpul3</th>\n",
       "      <th>jenpul4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.082569</td>\n",
       "      <td>967000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.054554</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.284404</td>\n",
       "      <td>190000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.010719</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.229358</td>\n",
       "      <td>9989100</td>\n",
       "      <td>0.005848</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.563547</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.458716</td>\n",
       "      <td>4485000</td>\n",
       "      <td>0.008772</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.253027</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.009174</td>\n",
       "      <td>2025500</td>\n",
       "      <td>0.001462</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.114271</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999995</th>\n",
       "      <td>0.458716</td>\n",
       "      <td>547100</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.030865</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999996</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>7193562</td>\n",
       "      <td>0.001462</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.405834</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999997</th>\n",
       "      <td>0.229358</td>\n",
       "      <td>3821317</td>\n",
       "      <td>0.001462</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.215584</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999998</th>\n",
       "      <td>0.220183</td>\n",
       "      <td>6801200</td>\n",
       "      <td>0.004386</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.383698</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999999</th>\n",
       "      <td>0.293578</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4000000 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             usia    biaya      nday     nproc    nsdiag  ismale  jenpel1  \\\n",
       "0        0.082569   967000  0.000000  0.045455  0.054554     1.0      0.0   \n",
       "1        0.284404   190000  0.000000  0.000000  0.010719     0.0      0.0   \n",
       "2        0.229358  9989100  0.005848  0.181818  0.563547     0.0      1.0   \n",
       "3        0.458716  4485000  0.008772  0.090909  0.253027     1.0      1.0   \n",
       "4        0.009174  2025500  0.001462  0.000000  0.114271     0.0      1.0   \n",
       "...           ...      ...       ...       ...       ...     ...      ...   \n",
       "3999995  0.458716   547100  0.000000  0.000000  0.030865     0.0      0.0   \n",
       "3999996  0.000000  7193562  0.001462  0.000000  0.405834     0.0      1.0   \n",
       "3999997  0.229358  3821317  0.001462  0.000000  0.215584     0.0      1.0   \n",
       "3999998  0.220183  6801200  0.004386  0.090909  0.383698     0.0      1.0   \n",
       "3999999  0.293578        0  0.000000  0.000000  0.000000     0.0      0.0   \n",
       "\n",
       "         jenpul1  jenpul2  jenpul3  jenpul4  \n",
       "0            1.0      0.0      0.0      0.0  \n",
       "1            1.0      0.0      0.0      0.0  \n",
       "2            0.0      0.0      0.0      1.0  \n",
       "3            1.0      0.0      0.0      0.0  \n",
       "4            1.0      0.0      0.0      0.0  \n",
       "...          ...      ...      ...      ...  \n",
       "3999995      1.0      0.0      0.0      0.0  \n",
       "3999996      0.0      0.0      0.0      0.0  \n",
       "3999997      1.0      0.0      0.0      0.0  \n",
       "3999998      1.0      0.0      0.0      0.0  \n",
       "3999999      1.0      0.0      0.0      0.0  \n",
       "\n",
       "[4000000 rows x 11 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# using minmax scaler to scale all column\n",
    "# save the fitted scaler to some folder, it usefull for reused in future implementation\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import joblib\n",
    "\n",
    "bpjs_mmscaler = MinMaxScaler()\n",
    "columns = x.drop([\"biaya\"], axis=1).columns\n",
    "\n",
    "bpjs_sscaler = bpjs_mmscaler.fit(x[columns])\n",
    "joblib.dump(bpjs_mmscaler, \"bpjs_mmcaler.save\")\n",
    "\n",
    "x[columns] = bpjs_mmscaler.transform(x[columns])\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation & Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7453771</th>\n",
       "      <td>677670</td>\n",
       "      <td>42</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>148400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7051322</th>\n",
       "      <td>6883102</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>192400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7255929</th>\n",
       "      <td>8531594</td>\n",
       "      <td>12</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>184600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3205207</th>\n",
       "      <td>4729487</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>192100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7226889</th>\n",
       "      <td>5778008</td>\n",
       "      <td>39</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>243500.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6530525</th>\n",
       "      <td>4949443</td>\n",
       "      <td>44</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>195600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1299232</th>\n",
       "      <td>4850222</td>\n",
       "      <td>32</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>187000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7674248</th>\n",
       "      <td>4603081</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>3142800.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8428805</th>\n",
       "      <td>10559502</td>\n",
       "      <td>0</td>\n",
       "      <td>L</td>\n",
       "      <td>1</td>\n",
       "      <td>3887600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9725554</th>\n",
       "      <td>10101757</td>\n",
       "      <td>3</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>189600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2258353 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id  usia jenkel  jenispel      biaya  jenispulang  nday  nproc  \\\n",
       "7453771    677670    42      P         2   148400.0          1.0     0    1.0   \n",
       "7051322   6883102     0      P         2   192400.0          1.0     0    0.0   \n",
       "7255929   8531594    12      L         2   184600.0          1.0     0    0.0   \n",
       "3205207   4729487    63      L         2   192100.0          1.0     0    1.0   \n",
       "7226889   5778008    39      P         2   243500.0          1.0     0    0.0   \n",
       "...           ...   ...    ...       ...        ...          ...   ...    ...   \n",
       "6530525   4949443    44      P         2   195600.0          1.0     0    0.0   \n",
       "1299232   4850222    32      L         2   187000.0          1.0     0    0.0   \n",
       "7674248   4603081     0      L         1  3142800.0          1.0     1    1.0   \n",
       "8428805  10559502     0      L         1  3887600.0          1.0     3    7.0   \n",
       "9725554  10101757     3      L         2   189600.0          1.0     0    0.0   \n",
       "\n",
       "         nsdiag  label  \n",
       "7453771     0.0      0  \n",
       "7051322     1.0      0  \n",
       "7255929     1.0      0  \n",
       "3205207     0.0      0  \n",
       "7226889     0.0      0  \n",
       "...         ...    ...  \n",
       "6530525     2.0      0  \n",
       "1299232     0.0      0  \n",
       "7674248     0.0      0  \n",
       "8428805     0.0      0  \n",
       "9725554     0.0      0  \n",
       "\n",
       "[2258353 rows x 10 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# im splitting test data into validation and test\n",
    "# set new variable \n",
    "d_test = pd.concat([xtest, ytest], axis=1)\n",
    "d_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7453771</th>\n",
       "      <td>42</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>148400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7051322</th>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>192400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7255929</th>\n",
       "      <td>12</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>184600.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3205207</th>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>192100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7226889</th>\n",
       "      <td>39</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>243500.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         usia jenkel  jenispel     biaya  jenispulang  nday  nproc  nsdiag  \\\n",
       "7453771    42      P         2  148400.0          1.0     0    1.0     0.0   \n",
       "7051322     0      P         2  192400.0          1.0     0    0.0     1.0   \n",
       "7255929    12      L         2  184600.0          1.0     0    0.0     1.0   \n",
       "3205207    63      L         2  192100.0          1.0     0    1.0     0.0   \n",
       "7226889    39      P         2  243500.0          1.0     0    0.0     0.0   \n",
       "\n",
       "         label  \n",
       "7453771      0  \n",
       "7051322      0  \n",
       "7255929      0  \n",
       "3205207      0  \n",
       "7226889      0  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop id column\n",
    "d_test.drop(\"id\", axis=1, inplace=True)\n",
    "d_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2227048\n",
       "1      31305\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check label\n",
    "d_test[\"label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no id column skip formating for id column to np.int64\n",
      "no typefaskes column skip formating for typefaskes column to str\n",
      "no kelasrawat column skip formating for kelasrawat column to str\n",
      "no typefaskes column skip encoding typefaskes\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>usia</th>\n",
       "      <th>biaya</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>label</th>\n",
       "      <th>ismale</th>\n",
       "      <th>jenpel1</th>\n",
       "      <th>jenpul1</th>\n",
       "      <th>jenpul2</th>\n",
       "      <th>jenpul3</th>\n",
       "      <th>jenpul4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7453771</th>\n",
       "      <td>42</td>\n",
       "      <td>148400.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7051322</th>\n",
       "      <td>0</td>\n",
       "      <td>192400.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7255929</th>\n",
       "      <td>12</td>\n",
       "      <td>184600.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3205207</th>\n",
       "      <td>63</td>\n",
       "      <td>192100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7226889</th>\n",
       "      <td>39</td>\n",
       "      <td>243500.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6530525</th>\n",
       "      <td>44</td>\n",
       "      <td>195600.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1299232</th>\n",
       "      <td>32</td>\n",
       "      <td>187000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7674248</th>\n",
       "      <td>0</td>\n",
       "      <td>3142800.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8428805</th>\n",
       "      <td>0</td>\n",
       "      <td>3887600.0</td>\n",
       "      <td>3</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9725554</th>\n",
       "      <td>3</td>\n",
       "      <td>189600.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2258353 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         usia      biaya  nday  nproc  nsdiag  label  ismale  jenpel1  \\\n",
       "7453771    42   148400.0     0    1.0     0.0      0       0        0   \n",
       "7051322     0   192400.0     0    0.0     1.0      0       0        0   \n",
       "7255929    12   184600.0     0    0.0     1.0      0       1        0   \n",
       "3205207    63   192100.0     0    1.0     0.0      0       1        0   \n",
       "7226889    39   243500.0     0    0.0     0.0      0       0        0   \n",
       "...       ...        ...   ...    ...     ...    ...     ...      ...   \n",
       "6530525    44   195600.0     0    0.0     2.0      0       0        0   \n",
       "1299232    32   187000.0     0    0.0     0.0      0       1        0   \n",
       "7674248     0  3142800.0     1    1.0     0.0      0       1        1   \n",
       "8428805     0  3887600.0     3    7.0     0.0      0       1        1   \n",
       "9725554     3   189600.0     0    0.0     0.0      0       1        0   \n",
       "\n",
       "         jenpul1  jenpul2  jenpul3  jenpul4  \n",
       "7453771        1        0        0        0  \n",
       "7051322        1        0        0        0  \n",
       "7255929        1        0        0        0  \n",
       "3205207        1        0        0        0  \n",
       "7226889        1        0        0        0  \n",
       "...          ...      ...      ...      ...  \n",
       "6530525        1        0        0        0  \n",
       "1299232        1        0        0        0  \n",
       "7674248        1        0        0        0  \n",
       "8428805        1        0        0        0  \n",
       "9725554        1        0        0        0  \n",
       "\n",
       "[2258353 rows x 12 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# one hot encoding\n",
    "d_test = encode_categorical(d_test)\n",
    "d_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>usia</th>\n",
       "      <th>biaya</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>label</th>\n",
       "      <th>ismale</th>\n",
       "      <th>jenpel1</th>\n",
       "      <th>jenpul1</th>\n",
       "      <th>jenpul2</th>\n",
       "      <th>jenpul3</th>\n",
       "      <th>jenpul4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7453771</th>\n",
       "      <td>0.385321</td>\n",
       "      <td>148400.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7051322</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>192400.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.641622e-08</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7255929</th>\n",
       "      <td>0.110092</td>\n",
       "      <td>184600.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.641622e-08</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3205207</th>\n",
       "      <td>0.577982</td>\n",
       "      <td>192100.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7226889</th>\n",
       "      <td>0.357798</td>\n",
       "      <td>243500.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6530525</th>\n",
       "      <td>0.403670</td>\n",
       "      <td>195600.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.128324e-07</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1299232</th>\n",
       "      <td>0.293578</td>\n",
       "      <td>187000.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7674248</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3142800.0</td>\n",
       "      <td>0.001462</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8428805</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3887600.0</td>\n",
       "      <td>0.004386</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9725554</th>\n",
       "      <td>0.027523</td>\n",
       "      <td>189600.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2258353 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             usia      biaya      nday     nproc        nsdiag  label  ismale  \\\n",
       "7453771  0.385321   148400.0  0.000000  0.045455  0.000000e+00      0     0.0   \n",
       "7051322  0.000000   192400.0  0.000000  0.000000  5.641622e-08      0     0.0   \n",
       "7255929  0.110092   184600.0  0.000000  0.000000  5.641622e-08      0     1.0   \n",
       "3205207  0.577982   192100.0  0.000000  0.045455  0.000000e+00      0     1.0   \n",
       "7226889  0.357798   243500.0  0.000000  0.000000  0.000000e+00      0     0.0   \n",
       "...           ...        ...       ...       ...           ...    ...     ...   \n",
       "6530525  0.403670   195600.0  0.000000  0.000000  1.128324e-07      0     0.0   \n",
       "1299232  0.293578   187000.0  0.000000  0.000000  0.000000e+00      0     1.0   \n",
       "7674248  0.000000  3142800.0  0.001462  0.045455  0.000000e+00      0     1.0   \n",
       "8428805  0.000000  3887600.0  0.004386  0.318182  0.000000e+00      0     1.0   \n",
       "9725554  0.027523   189600.0  0.000000  0.000000  0.000000e+00      0     1.0   \n",
       "\n",
       "         jenpel1  jenpul1  jenpul2  jenpul3  jenpul4  \n",
       "7453771      0.0      1.0      0.0      0.0      0.0  \n",
       "7051322      0.0      1.0      0.0      0.0      0.0  \n",
       "7255929      0.0      1.0      0.0      0.0      0.0  \n",
       "3205207      0.0      1.0      0.0      0.0      0.0  \n",
       "7226889      0.0      1.0      0.0      0.0      0.0  \n",
       "...          ...      ...      ...      ...      ...  \n",
       "6530525      0.0      1.0      0.0      0.0      0.0  \n",
       "1299232      0.0      1.0      0.0      0.0      0.0  \n",
       "7674248      1.0      1.0      0.0      0.0      0.0  \n",
       "8428805      1.0      1.0      0.0      0.0      0.0  \n",
       "9725554      0.0      1.0      0.0      0.0      0.0  \n",
       "\n",
       "[2258353 rows x 12 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# scaling\n",
    "# columns = d_test.drop(\"label\", axis=1).columns\n",
    "d_test[columns] = bpjs_mmscaler.transform(d_test[columns])\n",
    "d_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split test data, 0.3 to validation, 0.7 to test\n",
    "xval, xtest, yval, ytest = train_test_split(d_test.drop(\"label\", axis=1), d_test[\"label\"], test_size=0.7, random_state=24)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    668009\n",
       "1      9496\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check validation label\n",
    "yval.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1559039\n",
       "1      21809\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check test label\n",
    "ytest.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# im using the NN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NN1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense (Dense)               (None, 18)                216       \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 1)                 19        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 235\n",
      "Trainable params: 235\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# create the neural network\n",
    "# im using 1 input, 1 hidden, and 1 output layer (1 node for 0-1 value)\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from keras import layers\n",
    "from keras.layers import BatchNormalization\n",
    "from sklearn.metrics import classification_report\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "nn = keras.Sequential()\n",
    "nn.add(layers.InputLayer(input_shape=(x.shape[1],)))\n",
    "nn.add(layers.Dense(18, activation=\"relu\"))\n",
    "nn.add(layers.Dense(1, activation=\"sigmoid\"))\n",
    "\n",
    "ckpt = ModelCheckpoint(\"model/nn_v2\", save_best_only=True)\n",
    "nn.compile(optimizer=\"adam\", loss=\"binary_crossentropy\", metrics=[\"accuracy\"])\n",
    "nn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "124990/125000 [============================>.] - ETA: 0s - loss: 581.4552 - accuracy: 0.6215INFO:tensorflow:Assets written to: model\\nn_v2\\assets\n",
      "125000/125000 [==============================] - 244s 2ms/step - loss: 581.4764 - accuracy: 0.6215 - val_loss: 31.1459 - val_accuracy: 0.9912\n",
      "Epoch 2/5\n",
      "125000/125000 [==============================] - 239s 2ms/step - loss: 325.4855 - accuracy: 0.6300 - val_loss: 693.0790 - val_accuracy: 0.0201\n",
      "Epoch 3/5\n",
      "125000/125000 [==============================] - 234s 2ms/step - loss: 239.9909 - accuracy: 0.6324 - val_loss: 391.9095 - val_accuracy: 0.0549\n",
      "Epoch 4/5\n",
      "124979/125000 [============================>.] - ETA: 0s - loss: 157.2824 - accuracy: 0.6344INFO:tensorflow:Assets written to: model\\nn_v2\\assets\n",
      "125000/125000 [==============================] - 238s 2ms/step - loss: 157.2908 - accuracy: 0.6344 - val_loss: 1.8612 - val_accuracy: 0.9912\n",
      "Epoch 5/5\n",
      "124972/125000 [============================>.] - ETA: 0s - loss: 68.1765 - accuracy: 0.6328INFO:tensorflow:Assets written to: model\\nn_v2\\assets\n",
      "125000/125000 [==============================] - 240s 2ms/step - loss: 68.1614 - accuracy: 0.6328 - val_loss: 0.5033 - val_accuracy: 0.9912\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1a7aefbf5b0>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# training the model using the artificial train data\n",
    "nn.fit(x, y, validation_data=(xval, yval), epochs=5, callbacks=ckpt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>usia</th>\n",
       "      <th>biaya</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>ismale</th>\n",
       "      <th>jenpel1</th>\n",
       "      <th>jenpul1</th>\n",
       "      <th>jenpul2</th>\n",
       "      <th>jenpul3</th>\n",
       "      <th>jenpul4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3986552</th>\n",
       "      <td>0.073394</td>\n",
       "      <td>182600.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>773816</th>\n",
       "      <td>0.522936</td>\n",
       "      <td>7582500.0</td>\n",
       "      <td>0.011696</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>2.256649e-07</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1022982</th>\n",
       "      <td>0.532110</td>\n",
       "      <td>409200.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>1.128324e-07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2338226</th>\n",
       "      <td>0.605505</td>\n",
       "      <td>204200.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.641622e-08</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11207355</th>\n",
       "      <td>0.348624</td>\n",
       "      <td>357600.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8159</th>\n",
       "      <td>0.477064</td>\n",
       "      <td>1650100.0</td>\n",
       "      <td>0.007310</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.641622e-08</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6457892</th>\n",
       "      <td>0.211009</td>\n",
       "      <td>183000.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9763342</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>5987500.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>1.692486e-07</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11265816</th>\n",
       "      <td>0.119266</td>\n",
       "      <td>186700.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8447687</th>\n",
       "      <td>0.596330</td>\n",
       "      <td>196100.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>5.641622e-08</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1580848 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              usia      biaya      nday     nproc        nsdiag  ismale  \\\n",
       "3986552   0.073394   182600.0  0.000000  0.000000  0.000000e+00     1.0   \n",
       "773816    0.522936  7582500.0  0.011696  0.363636  2.256649e-07     1.0   \n",
       "1022982   0.532110   409200.0  0.000000  0.090909  1.128324e-07     0.0   \n",
       "2338226   0.605505   204200.0  0.000000  0.000000  5.641622e-08     1.0   \n",
       "11207355  0.348624   357600.0  0.000000  0.000000  0.000000e+00     1.0   \n",
       "...            ...        ...       ...       ...           ...     ...   \n",
       "8159      0.477064  1650100.0  0.007310  0.000000  5.641622e-08     0.0   \n",
       "6457892   0.211009   183000.0  0.000000  0.000000  0.000000e+00     1.0   \n",
       "9763342   0.000000  5987500.0  0.000000  0.090909  1.692486e-07     1.0   \n",
       "11265816  0.119266   186700.0  0.000000  0.000000  0.000000e+00     0.0   \n",
       "8447687   0.596330   196100.0  0.000000  0.045455  5.641622e-08     0.0   \n",
       "\n",
       "          jenpel1  jenpul1  jenpul2  jenpul3  jenpul4  \n",
       "3986552       0.0      1.0      0.0      0.0      0.0  \n",
       "773816        1.0      1.0      0.0      0.0      0.0  \n",
       "1022982       0.0      1.0      0.0      0.0      0.0  \n",
       "2338226       0.0      1.0      0.0      0.0      0.0  \n",
       "11207355      0.0      1.0      0.0      0.0      0.0  \n",
       "...           ...      ...      ...      ...      ...  \n",
       "8159          1.0      1.0      0.0      0.0      0.0  \n",
       "6457892       0.0      1.0      0.0      0.0      0.0  \n",
       "9763342       1.0      1.0      0.0      0.0      0.0  \n",
       "11265816      0.0      1.0      0.0      0.0      0.0  \n",
       "8447687       0.0      0.0      1.0      0.0      0.0  \n",
       "\n",
       "[1580848 rows x 11 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make sure the encoding and scalling is correct\n",
    "xtest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49402/49402 [==============================] - 57s 1ms/step\n"
     ]
    }
   ],
   "source": [
    "# predict the xtest\n",
    "ypred = nn.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# because the output is continous from 0-1 im round it so it will only have 0 or 1 match for label\n",
    "ypred = np.round(ypred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00   1559039\n",
      "           1       0.95      0.38      0.55     21809\n",
      "\n",
      "    accuracy                           0.99   1580848\n",
      "   macro avg       0.97      0.69      0.77   1580848\n",
      "weighted avg       0.99      0.99      0.99   1580848\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#print the classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(ytest, ypred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get information of the confussion matrix for calculating specificity\n",
    "from sklearn.metrics import confusion_matrix\n",
    "tn, fp, fn, tp = confusion_matrix(ytest, ypred).ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9997318861170247"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculating specificity\n",
    "tn/(tn+fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the model to some folder\n",
    "nn.save(\"model/nn_v2.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.01      0.01   1559039\n",
      "           1       0.01      1.00      0.03     21809\n",
      "\n",
      "    accuracy                           0.02   1580848\n",
      "   macro avg       0.50      0.50      0.02   1580848\n",
      "weighted avg       0.98      0.02      0.01   1580848\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "dt = DecisionTreeClassifier()\n",
    "dt.fit(x, y)\n",
    "ypred = dt.predict(xtest)\n",
    "print(classification_report(ytest, ypred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0050871081480322175"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn, fp, fn, tp = confusion_matrix(ytest, ypred).ravel()\n",
    "\n",
    "tn/(tn+fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.03      0.06   1559039\n",
      "           1       0.01      0.99      0.03     21809\n",
      "\n",
      "    accuracy                           0.04   1580848\n",
      "   macro avg       0.50      0.51      0.04   1580848\n",
      "weighted avg       0.98      0.04      0.06   1580848\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier()\n",
    "rf.fit(x, y)\n",
    "ypred = rf.predict(xtest)\n",
    "print(classification_report(ytest, ypred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.031152524086953567"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn, fp, fn, tp = confusion_matrix(ytest, ypred).ravel()\n",
    "\n",
    "tn/(tn+fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99   1559039\n",
      "           1       0.96      0.20      0.33     21809\n",
      "\n",
      "    accuracy                           0.99   1580848\n",
      "   macro avg       0.98      0.60      0.66   1580848\n",
      "weighted avg       0.99      0.99      0.99   1580848\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression(max_iter=1000)\n",
    "\n",
    "lr.fit(x, y)\n",
    "ypred = lr.predict(xtest)\n",
    "print(classification_report(ytest, ypred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9998890342063284"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn, fp, fn, tp = confusion_matrix(ytest, ypred).ravel()\n",
    "\n",
    "tn/(tn+fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\ewwwr\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\svm\\_base.py:1225: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n",
      "c:\\Users\\ewwwr\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "c:\\Users\\ewwwr\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00   1559039\n",
      "           1       0.01      1.00      0.03     21809\n",
      "\n",
      "    accuracy                           0.01   1580848\n",
      "   macro avg       0.01      0.50      0.01   1580848\n",
      "weighted avg       0.00      0.01      0.00   1580848\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\ewwwr\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1334: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "svm = LinearSVC()\n",
    "\n",
    "svm.fit(x, y)\n",
    "ypred = svm.predict(xtest)\n",
    "print(classification_report(ytest, ypred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn, fp, fn, tp = confusion_matrix(ytest, ypred).ravel()\n",
    "\n",
    "tn/(tn+fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.89      0.94   1559039\n",
      "           1       0.07      0.64      0.13     21809\n",
      "\n",
      "    accuracy                           0.88   1580848\n",
      "   macro avg       0.53      0.76      0.53   1580848\n",
      "weighted avg       0.98      0.88      0.93   1580848\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "xgbc = XGBClassifier()\n",
    "xgbc.fit(x, y)\n",
    "ypred = xgbc.predict(xtest)\n",
    "print(classification_report(ytest, ypred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8865057256425273"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn, fp, fn, tp = confusion_matrix(ytest, ypred).ravel()\n",
    "\n",
    "tn/(tn+fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.37      0.54   1559039\n",
      "           1       0.02      0.74      0.03     21809\n",
      "\n",
      "    accuracy                           0.37   1580848\n",
      "   macro avg       0.50      0.55      0.28   1580848\n",
      "weighted avg       0.98      0.37      0.53   1580848\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "mnb = MultinomialNB()\n",
    "mnb.fit(x, y)\n",
    "ypred = mnb.predict(xtest)\n",
    "print(classification_report(ytest, ypred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3666406036026039"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn, fp, fn, tp = confusion_matrix(ytest, ypred).ravel()\n",
    "\n",
    "tn/(tn+fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-Nearest Neighbours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "# knn = KNeighborsClassifier()\n",
    "# knn.fit(x, y)\n",
    "# ypred = knn.predict(xtest)\n",
    "# print(classification_report(ytest, ypred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tn, fp, fn, tp = confusion_matrix(ytest, ypred).ravel()\n",
    "\n",
    "# tn/(tn+fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VotingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.ensemble import VotingClassifier\n",
    "# from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# vc = VotingClassifier([\n",
    "#     (\"nn\", nn),\n",
    "#     (\"dt\", dt),\n",
    "#     (\"svm\", svm)\n",
    "# ])\n",
    "# params = {\n",
    "#     \"voting\":[\"hard\", \"soft\"],\n",
    "#     \"weights\":[(1,1,1),(2,1,1),(1,2,1),(1,1,2),(2,2,1),(1,2,2)]\n",
    "# }\n",
    "# grid = GridSearchCV(vc, params)\n",
    "\n",
    "# grid.fit(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(grid.best_params_)\n",
    "# print(grid.best_score_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# --Submission--"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>pisat</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>...</th>\n",
       "      <th>diagfktp</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "      <th>kelasrawat</th>\n",
       "      <th>kdsa</th>\n",
       "      <th>kdsp</th>\n",
       "      <th>kdsr</th>\n",
       "      <th>kdsi</th>\n",
       "      <th>kdsd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>KL</td>\n",
       "      <td>67</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-01-20T17:00:00.000Z</td>\n",
       "      <td>2020-01-20T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>K21.9</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>60</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-02-06T17:00:00.000Z</td>\n",
       "      <td>2019-02-06T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>I63</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-12-11T17:00:00.000Z</td>\n",
       "      <td>2019-12-11T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>E10.9</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>52</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-02-24T17:00:00.000Z</td>\n",
       "      <td>2020-02-24T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>M54.5</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2020-02-05T17:00:00.000Z</td>\n",
       "      <td>2020-02-05T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>I11.9</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>455269</td>\n",
       "      <td>1012063</td>\n",
       "      <td>233</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>P03.4</td>\n",
       "      <td>3457000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>1006349</td>\n",
       "      <td>1012064</td>\n",
       "      <td>241</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-01T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>P03.4</td>\n",
       "      <td>3416200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>1093249</td>\n",
       "      <td>1012065</td>\n",
       "      <td>318</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2021-01-01T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>P03.4</td>\n",
       "      <td>3762200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>1003715</td>\n",
       "      <td>1012066</td>\n",
       "      <td>303</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-22T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>Z03.8</td>\n",
       "      <td>9672000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-08-III</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>2226</td>\n",
       "      <td>1012067</td>\n",
       "      <td>5</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-04T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>A09.9</td>\n",
       "      <td>1465800.0</td>\n",
       "      <td>1</td>\n",
       "      <td>K-4-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  id_peserta  dati2 typefaskes  usia jenkel  pisat  \\\n",
       "0           3548           1      8         KL    67      L    1.0   \n",
       "1          85390           2     17         A     60      L    1.0   \n",
       "2          89574           3     17         B     66      P    1.0   \n",
       "3          88653           4     17         B     52      P    1.0   \n",
       "4          88634           5     17         B     63      L    2.0   \n",
       "...          ...         ...    ...        ...   ...    ...    ...   \n",
       "1099333   455269     1012063    233         SC     0      P    4.0   \n",
       "1099334  1006349     1012064    241         C      0      P    4.0   \n",
       "1099335  1093249     1012065    318         B      0      P    4.0   \n",
       "1099336  1003715     1012066    303         B      0      P    4.0   \n",
       "1099337     2226     1012067      5         SC     0      P    4.0   \n",
       "\n",
       "                        tgldatang                 tglpulang  jenispel  ...  \\\n",
       "0        2020-01-20T17:00:00.000Z  2020-01-20T17:00:00.000Z         2  ...   \n",
       "1        2019-02-06T17:00:00.000Z  2019-02-06T17:00:00.000Z         2  ...   \n",
       "2        2019-12-11T17:00:00.000Z  2019-12-11T17:00:00.000Z         2  ...   \n",
       "3        2020-02-24T17:00:00.000Z  2020-02-24T17:00:00.000Z         2  ...   \n",
       "4        2020-02-05T17:00:00.000Z  2020-02-05T17:00:00.000Z         2  ...   \n",
       "...                           ...                       ...       ...  ...   \n",
       "1099333  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1  ...   \n",
       "1099334  2020-12-30T17:00:00.000Z  2021-01-01T17:00:00.000Z         1  ...   \n",
       "1099335  2020-12-29T17:00:00.000Z  2021-01-01T17:00:00.000Z         1  ...   \n",
       "1099336  2020-12-30T17:00:00.000Z  2021-01-22T17:00:00.000Z         1  ...   \n",
       "1099337  2020-12-30T17:00:00.000Z  2021-01-04T17:00:00.000Z         1  ...   \n",
       "\n",
       "        diagfktp      biaya  jenispulang         cbg kelasrawat  kdsa  kdsp  \\\n",
       "0          K21.9   187300.0            1    Q-5-44-0          3  None  None   \n",
       "1            I63   381600.0            1    Q-5-44-0          3  None  None   \n",
       "2          E10.9   185500.0            1    Q-5-44-0          3  None  None   \n",
       "3          M54.5   185500.0            1    Q-5-44-0          3  None  None   \n",
       "4          I11.9   185500.0            1    Q-5-44-0          3  None  None   \n",
       "...          ...        ...          ...         ...        ...   ...   ...   \n",
       "1099333    P03.4  3457000.0            1    P-8-17-I          3  None  None   \n",
       "1099334    P03.4  3416200.0            1    P-8-17-I          3  None  None   \n",
       "1099335    P03.4  3762200.0            1    P-8-17-I          3  None  None   \n",
       "1099336    Z03.8  9672000.0            1  P-8-08-III          3  None  None   \n",
       "1099337    A09.9  1465800.0            1    K-4-17-I          3  None  None   \n",
       "\n",
       "         kdsr  kdsi  kdsd  \n",
       "0        None  None  None  \n",
       "1        None  None  None  \n",
       "2        None  None  None  \n",
       "3        None  None  None  \n",
       "4        None  None  None  \n",
       "...       ...   ...   ...  \n",
       "1099333  None  None  None  \n",
       "1099334  None  None  None  \n",
       "1099335  None  None  None  \n",
       "1099336  None  None  None  \n",
       "1099337  None  None  None  \n",
       "\n",
       "[1099338 rows x 21 columns]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading sampling\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "samp = pd.read_csv(\"dataset/test/sampling3_healthkathon2022_sep.csv\", sep=\";\")\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>kdproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>74.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869697</th>\n",
       "      <td>1099314</td>\n",
       "      <td>73.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869698</th>\n",
       "      <td>1099318</td>\n",
       "      <td>90.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869699</th>\n",
       "      <td>1099318</td>\n",
       "      <td>87.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869700</th>\n",
       "      <td>1099322</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869701</th>\n",
       "      <td>1099337</td>\n",
       "      <td>90.54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>869702 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id kdproc\n",
       "0             2   74.0\n",
       "1             2  88.78\n",
       "2             2  90.59\n",
       "3             7  90.59\n",
       "4            10  90.59\n",
       "...         ...    ...\n",
       "869697  1099314  73.59\n",
       "869698  1099318  90.46\n",
       "869699  1099318  87.44\n",
       "869700  1099322  88.78\n",
       "869701  1099337  90.54\n",
       "\n",
       "[869702 rows x 2 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading proc\n",
    "proc = pd.read_csv(\"dataset/test/sampling3_healthkathon2022_procedure.csv\", sep=\";\")\n",
    "proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>kddiag</th>\n",
       "      <th>levelid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>437398</td>\n",
       "      <td>F90.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>437401</td>\n",
       "      <td>H25.9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>438542</td>\n",
       "      <td>A16.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>439298</td>\n",
       "      <td>Z09.8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>439749</td>\n",
       "      <td>O48</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209467</th>\n",
       "      <td>794601</td>\n",
       "      <td>H66.9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209468</th>\n",
       "      <td>795524</td>\n",
       "      <td>O99.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209469</th>\n",
       "      <td>797586</td>\n",
       "      <td>H04.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209470</th>\n",
       "      <td>798050</td>\n",
       "      <td>N40</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209471</th>\n",
       "      <td>798167</td>\n",
       "      <td>J45.9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2209472 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id kddiag  levelid\n",
       "0        437398  F90.0        2\n",
       "1        437401  H25.9        1\n",
       "2        438542  A16.0        2\n",
       "3        439298  Z09.8        1\n",
       "4        439749    O48        1\n",
       "...         ...    ...      ...\n",
       "2209467  794601  H66.9        2\n",
       "2209468  795524  O99.0        1\n",
       "2209469  797586  H04.0        2\n",
       "2209470  798050    N40        2\n",
       "2209471  798167  J45.9        1\n",
       "\n",
       "[2209472 rows x 3 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading diag\n",
    "diag = pd.read_csv(\"dataset/test/sampling3_healthkathon2022_diagnosa.csv\", sep=\";\")\n",
    "diag"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleansing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>pisat</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>...</th>\n",
       "      <th>diagfktp</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "      <th>kelasrawat</th>\n",
       "      <th>kdsa</th>\n",
       "      <th>kdsp</th>\n",
       "      <th>kdsr</th>\n",
       "      <th>kdsi</th>\n",
       "      <th>kdsd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>KL</td>\n",
       "      <td>67</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-01-20T17:00:00.000Z</td>\n",
       "      <td>2020-01-20T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>K21.9</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>60</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-02-06T17:00:00.000Z</td>\n",
       "      <td>2019-02-06T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>I63</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-12-11T17:00:00.000Z</td>\n",
       "      <td>2019-12-11T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>E10.9</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>52</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-02-24T17:00:00.000Z</td>\n",
       "      <td>2020-02-24T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>M54.5</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2020-02-05T17:00:00.000Z</td>\n",
       "      <td>2020-02-05T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>I11.9</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>455269</td>\n",
       "      <td>1012063</td>\n",
       "      <td>233</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>P03.4</td>\n",
       "      <td>3457000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>1006349</td>\n",
       "      <td>1012064</td>\n",
       "      <td>241</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-01T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>P03.4</td>\n",
       "      <td>3416200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>1093249</td>\n",
       "      <td>1012065</td>\n",
       "      <td>318</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2021-01-01T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>P03.4</td>\n",
       "      <td>3762200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>1003715</td>\n",
       "      <td>1012066</td>\n",
       "      <td>303</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-22T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>Z03.8</td>\n",
       "      <td>9672000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-08-III</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>2226</td>\n",
       "      <td>1012067</td>\n",
       "      <td>5</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-04T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>A09.9</td>\n",
       "      <td>1465800.0</td>\n",
       "      <td>1</td>\n",
       "      <td>K-4-17-I</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  id_peserta  dati2 typefaskes  usia jenkel  pisat  \\\n",
       "0           3548           1      8         KL    67      L    1.0   \n",
       "1          85390           2     17         A     60      L    1.0   \n",
       "2          89574           3     17         B     66      P    1.0   \n",
       "3          88653           4     17         B     52      P    1.0   \n",
       "4          88634           5     17         B     63      L    2.0   \n",
       "...          ...         ...    ...        ...   ...    ...    ...   \n",
       "1099333   455269     1012063    233         SC     0      P    4.0   \n",
       "1099334  1006349     1012064    241         C      0      P    4.0   \n",
       "1099335  1093249     1012065    318         B      0      P    4.0   \n",
       "1099336  1003715     1012066    303         B      0      P    4.0   \n",
       "1099337     2226     1012067      5         SC     0      P    4.0   \n",
       "\n",
       "                        tgldatang                 tglpulang  jenispel  ...  \\\n",
       "0        2020-01-20T17:00:00.000Z  2020-01-20T17:00:00.000Z         2  ...   \n",
       "1        2019-02-06T17:00:00.000Z  2019-02-06T17:00:00.000Z         2  ...   \n",
       "2        2019-12-11T17:00:00.000Z  2019-12-11T17:00:00.000Z         2  ...   \n",
       "3        2020-02-24T17:00:00.000Z  2020-02-24T17:00:00.000Z         2  ...   \n",
       "4        2020-02-05T17:00:00.000Z  2020-02-05T17:00:00.000Z         2  ...   \n",
       "...                           ...                       ...       ...  ...   \n",
       "1099333  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1  ...   \n",
       "1099334  2020-12-30T17:00:00.000Z  2021-01-01T17:00:00.000Z         1  ...   \n",
       "1099335  2020-12-29T17:00:00.000Z  2021-01-01T17:00:00.000Z         1  ...   \n",
       "1099336  2020-12-30T17:00:00.000Z  2021-01-22T17:00:00.000Z         1  ...   \n",
       "1099337  2020-12-30T17:00:00.000Z  2021-01-04T17:00:00.000Z         1  ...   \n",
       "\n",
       "        diagfktp      biaya  jenispulang         cbg kelasrawat  kdsa  kdsp  \\\n",
       "0          K21.9   187300.0            1    Q-5-44-0          3  None  None   \n",
       "1            I63   381600.0            1    Q-5-44-0          3  None  None   \n",
       "2          E10.9   185500.0            1    Q-5-44-0          3  None  None   \n",
       "3          M54.5   185500.0            1    Q-5-44-0          3  None  None   \n",
       "4          I11.9   185500.0            1    Q-5-44-0          3  None  None   \n",
       "...          ...        ...          ...         ...        ...   ...   ...   \n",
       "1099333    P03.4  3457000.0            1    P-8-17-I          3  None  None   \n",
       "1099334    P03.4  3416200.0            1    P-8-17-I          3  None  None   \n",
       "1099335    P03.4  3762200.0            1    P-8-17-I          3  None  None   \n",
       "1099336    Z03.8  9672000.0            1  P-8-08-III          3  None  None   \n",
       "1099337    A09.9  1465800.0            1    K-4-17-I          3  None  None   \n",
       "\n",
       "         kdsr  kdsi  kdsd  \n",
       "0        None  None  None  \n",
       "1        None  None  None  \n",
       "2        None  None  None  \n",
       "3        None  None  None  \n",
       "4        None  None  None  \n",
       "...       ...   ...   ...  \n",
       "1099333  None  None  None  \n",
       "1099334  None  None  None  \n",
       "1099335  None  None  None  \n",
       "1099336  None  None  None  \n",
       "1099337  None  None  None  \n",
       "\n",
       "[1099338 rows x 21 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading samp\n",
    "import pandas as pd\n",
    "import gc\n",
    "samp = pd.read_csv(\"dataset/test/sampling3_healthkathon2022_sep.csv\", sep=\";\")\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dropping duplicate id because it must be unique\n",
    "# samp.drop_duplicates(subset=\"id\", inplace=True)\n",
    "# samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                  0\n",
       "id_peserta          0\n",
       "dati2               0\n",
       "typefaskes          0\n",
       "usia                0\n",
       "jenkel              2\n",
       "pisat               7\n",
       "tgldatang           0\n",
       "tglpulang           0\n",
       "jenispel            0\n",
       "politujuan     117900\n",
       "diagfktp           77\n",
       "biaya            5530\n",
       "jenispulang         0\n",
       "cbg                 0\n",
       "kelasrawat          0\n",
       "kdsa            23348\n",
       "kdsp            23276\n",
       "kdsr            23344\n",
       "kdsi            23325\n",
       "kdsd            23329\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if there is any null value\n",
    "samp.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dati2: 486 unique\n",
      "typefaskes: 26 unique\n",
      "usia: 106 unique\n",
      "jenkel: 3 unique\n",
      "pisat: 6 unique\n",
      "tgldatang: 1096 unique\n",
      "tglpulang: 1118 unique\n",
      "jenispel: 2 unique\n",
      "biaya: 18282 unique\n",
      "jenispulang: 5 unique\n",
      "cbg: 883 unique\n",
      "kelasrawat: 3 unique\n"
     ]
    }
   ],
   "source": [
    "# check the amount of unique value each row\n",
    "for i in samp.columns:\n",
    "    if i not in [\"id\", \"id_peserta\", \"diagfktp\", \"label\", \"kdsa\", \"kdsp\", \"kdsr\", \"kdsi\", \"kdsd\", \"politujuan\"]:\n",
    "        print(f\"{i}: {len(samp[i].unique())} unique\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "id_peserta     0\n",
       "dati2          0\n",
       "typefaskes     0\n",
       "usia           0\n",
       "jenkel         2\n",
       "pisat          7\n",
       "tgldatang      0\n",
       "tglpulang      0\n",
       "jenispel       0\n",
       "biaya          0\n",
       "jenispulang    0\n",
       "cbg            0\n",
       "kelasrawat     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# droping too many unique value and in the kdsa, kdsp, ... there is many null value \n",
    "# but it filled with string \"None\"\n",
    "samp.drop([\"politujuan\", \"diagfktp\", \"kdsa\", \"kdsp\", \"kdsr\", \"kdsi\", \"kdsd\"], axis=1, inplace=True)\n",
    "samp[\"biaya\"].fillna(0, inplace=True)\n",
    "samp.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "id_peserta     0\n",
       "dati2          0\n",
       "typefaskes     0\n",
       "usia           0\n",
       "jenkel         0\n",
       "pisat          0\n",
       "tgldatang      0\n",
       "tglpulang      0\n",
       "jenispel       0\n",
       "biaya          0\n",
       "jenispulang    0\n",
       "cbg            0\n",
       "kelasrawat     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samp[\"jenkel\"].fillna(samp[\"jenkel\"].mode()[0], inplace=True)\n",
    "samp[\"pisat\"].fillna(samp[\"pisat\"].mode()[0], inplace=True)\n",
    "samp.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>pisat</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "      <th>kelasrawat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>KL</td>\n",
       "      <td>67</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-01-20T17:00:00.000Z</td>\n",
       "      <td>2020-01-20T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>60</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-02-06T17:00:00.000Z</td>\n",
       "      <td>2019-02-06T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-12-11T17:00:00.000Z</td>\n",
       "      <td>2019-12-11T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>52</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-02-24T17:00:00.000Z</td>\n",
       "      <td>2020-02-24T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2020-02-05T17:00:00.000Z</td>\n",
       "      <td>2020-02-05T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>455269</td>\n",
       "      <td>1012063</td>\n",
       "      <td>233</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3457000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>1006349</td>\n",
       "      <td>1012064</td>\n",
       "      <td>241</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-01T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3416200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>1093249</td>\n",
       "      <td>1012065</td>\n",
       "      <td>318</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2021-01-01T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3762200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>1003715</td>\n",
       "      <td>1012066</td>\n",
       "      <td>303</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-22T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>9672000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-08-III</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>2226</td>\n",
       "      <td>1012067</td>\n",
       "      <td>5</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-04T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>1465800.0</td>\n",
       "      <td>1</td>\n",
       "      <td>K-4-17-I</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  id_peserta  dati2 typefaskes  usia jenkel  pisat  \\\n",
       "0           3548           1      8         KL    67      L    1.0   \n",
       "1          85390           2     17          A    60      L    1.0   \n",
       "2          89574           3     17          B    66      P    1.0   \n",
       "3          88653           4     17          B    52      P    1.0   \n",
       "4          88634           5     17          B    63      L    2.0   \n",
       "...          ...         ...    ...        ...   ...    ...    ...   \n",
       "1099333   455269     1012063    233         SC     0      P    4.0   \n",
       "1099334  1006349     1012064    241          C     0      P    4.0   \n",
       "1099335  1093249     1012065    318          B     0      P    4.0   \n",
       "1099336  1003715     1012066    303          B     0      P    4.0   \n",
       "1099337     2226     1012067      5         SC     0      P    4.0   \n",
       "\n",
       "                        tgldatang                 tglpulang  jenispel  \\\n",
       "0        2020-01-20T17:00:00.000Z  2020-01-20T17:00:00.000Z         2   \n",
       "1        2019-02-06T17:00:00.000Z  2019-02-06T17:00:00.000Z         2   \n",
       "2        2019-12-11T17:00:00.000Z  2019-12-11T17:00:00.000Z         2   \n",
       "3        2020-02-24T17:00:00.000Z  2020-02-24T17:00:00.000Z         2   \n",
       "4        2020-02-05T17:00:00.000Z  2020-02-05T17:00:00.000Z         2   \n",
       "...                           ...                       ...       ...   \n",
       "1099333  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1   \n",
       "1099334  2020-12-30T17:00:00.000Z  2021-01-01T17:00:00.000Z         1   \n",
       "1099335  2020-12-29T17:00:00.000Z  2021-01-01T17:00:00.000Z         1   \n",
       "1099336  2020-12-30T17:00:00.000Z  2021-01-22T17:00:00.000Z         1   \n",
       "1099337  2020-12-30T17:00:00.000Z  2021-01-04T17:00:00.000Z         1   \n",
       "\n",
       "             biaya  jenispulang         cbg  kelasrawat  \n",
       "0         187300.0            1    Q-5-44-0           3  \n",
       "1         381600.0            1    Q-5-44-0           3  \n",
       "2         185500.0            1    Q-5-44-0           3  \n",
       "3         185500.0            1    Q-5-44-0           3  \n",
       "4         185500.0            1    Q-5-44-0           3  \n",
       "...            ...          ...         ...         ...  \n",
       "1099333  3457000.0            1    P-8-17-I           3  \n",
       "1099334  3416200.0            1    P-8-17-I           3  \n",
       "1099335  3762200.0            1    P-8-17-I           3  \n",
       "1099336  9672000.0            1  P-8-08-III           3  \n",
       "1099337  1465800.0            1    K-4-17-I           3  \n",
       "\n",
       "[1099338 rows x 14 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tidy up all string column to uppercase and remove whitespace\n",
    "samp[\"typefaskes\"] = samp[\"typefaskes\"].str.upper()\n",
    "samp[\"typefaskes\"] = samp[\"typefaskes\"].str.replace(\" \", \"\")\n",
    "samp[\"jenkel\"] = samp[\"jenkel\"].str.upper()\n",
    "samp[\"jenkel\"] = samp[\"jenkel\"].str.replace(\" \", \"\")\n",
    "samp[\"cbg\"] = samp[\"cbg\"].str.upper()\n",
    "samp[\"cbg\"] = samp[\"cbg\"].str.replace(\" \", \"\")\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save cleaned sampling to some folder\n",
    "samp.to_csv(\"temp_data/1/test_samp_cleaned.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>kdproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>74.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869697</th>\n",
       "      <td>1099314</td>\n",
       "      <td>73.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869698</th>\n",
       "      <td>1099318</td>\n",
       "      <td>90.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869699</th>\n",
       "      <td>1099318</td>\n",
       "      <td>87.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869700</th>\n",
       "      <td>1099322</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869701</th>\n",
       "      <td>1099337</td>\n",
       "      <td>90.54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>869702 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id kdproc\n",
       "0             2   74.0\n",
       "1             2  88.78\n",
       "2             2  90.59\n",
       "3             7  90.59\n",
       "4            10  90.59\n",
       "...         ...    ...\n",
       "869697  1099314  73.59\n",
       "869698  1099318  90.46\n",
       "869699  1099318  87.44\n",
       "869700  1099322  88.78\n",
       "869701  1099337  90.54\n",
       "\n",
       "[869702 rows x 2 columns]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading procedure dataset\n",
    "import pandas as pd\n",
    "import gc\n",
    "proc = pd.read_csv(\"dataset/test/sampling3_healthkathon2022_procedure.csv\", sep=\";\")\n",
    "proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id        0\n",
       "kdproc    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if theres any null value\n",
    "proc.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rename column name\n",
    "proc.rename(columns={\"kdproc\":\"proc\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "171. - could not convert string to float: '-'\n",
      "\t10915\n",
      "1957. A59.8 could not convert string to float: 'A59.8'\n",
      "\t1\n",
      "1963. J45.9 could not convert string to float: 'J45.9'\n",
      "\t1\n",
      "2231. K30 could not convert string to float: 'K30'\n",
      "\t1\n"
     ]
    }
   ],
   "source": [
    "# check if there is any inconsistent value\n",
    "# result there is inconsistent value\n",
    "for index, val in enumerate(proc[\"proc\"].unique().tolist()):\n",
    "    try:\n",
    "        float(val)\n",
    "    except Exception as e:\n",
    "        print(f\"{index}. {val} {e}\")\n",
    "        print(f\"\\t{len(proc[proc['proc'] == val])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1956. A59.8 could not convert string to float: 'A59.8'\n",
      "\t1\n",
      "1962. J45.9 could not convert string to float: 'J45.9'\n",
      "\t1\n",
      "2230. K30 could not convert string to float: 'K30'\n",
      "\t1\n"
     ]
    }
   ],
   "source": [
    "# dropping that has \"-\" value\n",
    "# keeping all other string because i think there is procedure but miss typed it\n",
    "# because im only using the amount of procedure its not a problem just count it as 1\n",
    "index_to_drop = list(proc[proc['proc'].str.contains('[-]', na=False)].index)\n",
    "proc.drop(index=index_to_drop, inplace=True)\n",
    "\n",
    "# check again is inconsitent value resolved\n",
    "for index, val in enumerate(proc[\"proc\"].unique().tolist()):\n",
    "    try:\n",
    "        float(val)\n",
    "    except Exception as e:\n",
    "        print(f\"{index}. {val} {e}\")\n",
    "        print(f\"\\t{len(proc[proc['proc'] == val])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>74.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869697</th>\n",
       "      <td>1099314</td>\n",
       "      <td>73.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869698</th>\n",
       "      <td>1099318</td>\n",
       "      <td>90.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869699</th>\n",
       "      <td>1099318</td>\n",
       "      <td>87.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869700</th>\n",
       "      <td>1099322</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869701</th>\n",
       "      <td>1099337</td>\n",
       "      <td>90.54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>858787 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id   proc\n",
       "0             2   74.0\n",
       "1             2  88.78\n",
       "2             2  90.59\n",
       "3             7  90.59\n",
       "4            10  90.59\n",
       "...         ...    ...\n",
       "869697  1099314  73.59\n",
       "869698  1099318  90.46\n",
       "869699  1099318  87.44\n",
       "869700  1099322  88.78\n",
       "869701  1099337  90.54\n",
       "\n",
       "[858787 rows x 2 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save to some folder\n",
    "proc.to_csv(\"temp_data/1/test_proc_cleaned.csv\", index=False)\n",
    "proc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>kddiag</th>\n",
       "      <th>levelid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>437398</td>\n",
       "      <td>F90.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>437401</td>\n",
       "      <td>H25.9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>438542</td>\n",
       "      <td>A16.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>439298</td>\n",
       "      <td>Z09.8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>439749</td>\n",
       "      <td>O48</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209467</th>\n",
       "      <td>794601</td>\n",
       "      <td>H66.9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209468</th>\n",
       "      <td>795524</td>\n",
       "      <td>O99.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209469</th>\n",
       "      <td>797586</td>\n",
       "      <td>H04.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209470</th>\n",
       "      <td>798050</td>\n",
       "      <td>N40</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209471</th>\n",
       "      <td>798167</td>\n",
       "      <td>J45.9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2209472 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id kddiag  levelid\n",
       "0        437398  F90.0        2\n",
       "1        437401  H25.9        1\n",
       "2        438542  A16.0        2\n",
       "3        439298  Z09.8        1\n",
       "4        439749    O48        1\n",
       "...         ...    ...      ...\n",
       "2209467  794601  H66.9        2\n",
       "2209468  795524  O99.0        1\n",
       "2209469  797586  H04.0        2\n",
       "2209470  798050    N40        2\n",
       "2209471  798167  J45.9        1\n",
       "\n",
       "[2209472 rows x 3 columns]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading diagnosa data\n",
    "import pandas as pd\n",
    "import gc\n",
    "diag = pd.read_csv(\"dataset/test/sampling3_healthkathon2022_diagnosa.csv\", sep=\";\")\n",
    "diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>kddiag</th>\n",
       "      <th>levelid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>437398</td>\n",
       "      <td>F90.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>438542</td>\n",
       "      <td>A16.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>440827</td>\n",
       "      <td>I11.9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>446053</td>\n",
       "      <td>I64</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>453562</td>\n",
       "      <td>I64</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209460</th>\n",
       "      <td>789463</td>\n",
       "      <td>K21.9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209464</th>\n",
       "      <td>792932</td>\n",
       "      <td>N18.9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209467</th>\n",
       "      <td>794601</td>\n",
       "      <td>H66.9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209469</th>\n",
       "      <td>797586</td>\n",
       "      <td>H04.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2209470</th>\n",
       "      <td>798050</td>\n",
       "      <td>N40</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1112323 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id kddiag  levelid\n",
       "0        437398  F90.0        2\n",
       "2        438542  A16.0        2\n",
       "5        440827  I11.9        2\n",
       "8        446053    I64        2\n",
       "14       453562    I64        2\n",
       "...         ...    ...      ...\n",
       "2209460  789463  K21.9        2\n",
       "2209464  792932  N18.9        2\n",
       "2209467  794601  H66.9        2\n",
       "2209469  797586  H04.0        2\n",
       "2209470  798050    N40        2\n",
       "\n",
       "[1112323 rows x 3 columns]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dropping all levelid 1 because im only use the amount of secondary diagnose\n",
    "# or the levelid 2 as a feature\n",
    "index_to_drop = []\n",
    "index_to_drop = diag[diag[\"levelid\"] == 1].index\n",
    "diag.drop(index=index_to_drop, inplace=True)\n",
    "diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uppercase al string in kddiag column and remove whitespace\n",
    "diag[\"kddiag\"] = diag[\"kddiag\"].str.upper()\n",
    "diag[\"dkdiag\"] = diag[\"kddiag\"].str.replace(\" \", \"\")\n",
    "diag.to_csv(\"temp_data/1/test_diag_cleaned.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploratory Data Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_peserta</th>\n",
       "      <th>dati2</th>\n",
       "      <th>typefaskes</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>pisat</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>cbg</th>\n",
       "      <th>kelasrawat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>KL</td>\n",
       "      <td>67</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-01-20T17:00:00.000Z</td>\n",
       "      <td>2020-01-20T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>A</td>\n",
       "      <td>60</td>\n",
       "      <td>L</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-02-06T17:00:00.000Z</td>\n",
       "      <td>2019-02-06T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-12-11T17:00:00.000Z</td>\n",
       "      <td>2019-12-11T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>52</td>\n",
       "      <td>P</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-02-24T17:00:00.000Z</td>\n",
       "      <td>2020-02-24T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>B</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2020-02-05T17:00:00.000Z</td>\n",
       "      <td>2020-02-05T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Q-5-44-0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>455269</td>\n",
       "      <td>1012063</td>\n",
       "      <td>233</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3457000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>1006349</td>\n",
       "      <td>1012064</td>\n",
       "      <td>241</td>\n",
       "      <td>C</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-01T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3416200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>1093249</td>\n",
       "      <td>1012065</td>\n",
       "      <td>318</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2021-01-01T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3762200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-17-I</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>1003715</td>\n",
       "      <td>1012066</td>\n",
       "      <td>303</td>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-22T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>9672000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>P-8-08-III</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>2226</td>\n",
       "      <td>1012067</td>\n",
       "      <td>5</td>\n",
       "      <td>SC</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-04T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>1465800.0</td>\n",
       "      <td>1</td>\n",
       "      <td>K-4-17-I</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  id_peserta  dati2 typefaskes  usia jenkel  pisat  \\\n",
       "0           3548           1      8         KL    67      L    1.0   \n",
       "1          85390           2     17          A    60      L    1.0   \n",
       "2          89574           3     17          B    66      P    1.0   \n",
       "3          88653           4     17          B    52      P    1.0   \n",
       "4          88634           5     17          B    63      L    2.0   \n",
       "...          ...         ...    ...        ...   ...    ...    ...   \n",
       "1099333   455269     1012063    233         SC     0      P    4.0   \n",
       "1099334  1006349     1012064    241          C     0      P    4.0   \n",
       "1099335  1093249     1012065    318          B     0      P    4.0   \n",
       "1099336  1003715     1012066    303          B     0      P    4.0   \n",
       "1099337     2226     1012067      5         SC     0      P    4.0   \n",
       "\n",
       "                        tgldatang                 tglpulang  jenispel  \\\n",
       "0        2020-01-20T17:00:00.000Z  2020-01-20T17:00:00.000Z         2   \n",
       "1        2019-02-06T17:00:00.000Z  2019-02-06T17:00:00.000Z         2   \n",
       "2        2019-12-11T17:00:00.000Z  2019-12-11T17:00:00.000Z         2   \n",
       "3        2020-02-24T17:00:00.000Z  2020-02-24T17:00:00.000Z         2   \n",
       "4        2020-02-05T17:00:00.000Z  2020-02-05T17:00:00.000Z         2   \n",
       "...                           ...                       ...       ...   \n",
       "1099333  2020-12-29T17:00:00.000Z  2020-12-31T17:00:00.000Z         1   \n",
       "1099334  2020-12-30T17:00:00.000Z  2021-01-01T17:00:00.000Z         1   \n",
       "1099335  2020-12-29T17:00:00.000Z  2021-01-01T17:00:00.000Z         1   \n",
       "1099336  2020-12-30T17:00:00.000Z  2021-01-22T17:00:00.000Z         1   \n",
       "1099337  2020-12-30T17:00:00.000Z  2021-01-04T17:00:00.000Z         1   \n",
       "\n",
       "             biaya  jenispulang         cbg  kelasrawat  \n",
       "0         187300.0            1    Q-5-44-0           3  \n",
       "1         381600.0            1    Q-5-44-0           3  \n",
       "2         185500.0            1    Q-5-44-0           3  \n",
       "3         185500.0            1    Q-5-44-0           3  \n",
       "4         185500.0            1    Q-5-44-0           3  \n",
       "...            ...          ...         ...         ...  \n",
       "1099333  3457000.0            1    P-8-17-I           3  \n",
       "1099334  3416200.0            1    P-8-17-I           3  \n",
       "1099335  3762200.0            1    P-8-17-I           3  \n",
       "1099336  9672000.0            1  P-8-08-III           3  \n",
       "1099337  1465800.0            1    K-4-17-I           3  \n",
       "\n",
       "[1099338 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading samp data\n",
    "import pandas as pd\n",
    "\n",
    "samp = pd.read_csv(\"temp_data/1/test_samp_cleaned.csv\")\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "id_peserta     0\n",
       "dati2          0\n",
       "typefaskes     0\n",
       "usia           0\n",
       "jenkel         0\n",
       "pisat          0\n",
       "tgldatang      0\n",
       "tglpulang      0\n",
       "jenispel       0\n",
       "biaya          0\n",
       "jenispulang    0\n",
       "cbg            0\n",
       "kelasrawat     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the null\n",
    "samp.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1099338 entries, 0 to 1099337\n",
      "Data columns (total 14 columns):\n",
      " #   Column       Non-Null Count    Dtype  \n",
      "---  ------       --------------    -----  \n",
      " 0   id           1099338 non-null  int64  \n",
      " 1   id_peserta   1099338 non-null  int64  \n",
      " 2   dati2        1099338 non-null  int64  \n",
      " 3   typefaskes   1099338 non-null  object \n",
      " 4   usia         1099338 non-null  int64  \n",
      " 5   jenkel       1099338 non-null  object \n",
      " 6   pisat        1099338 non-null  float64\n",
      " 7   tgldatang    1099338 non-null  object \n",
      " 8   tglpulang    1099338 non-null  object \n",
      " 9   jenispel     1099338 non-null  int64  \n",
      " 10  biaya        1099338 non-null  float64\n",
      " 11  jenispulang  1099338 non-null  int64  \n",
      " 12  cbg          1099338 non-null  object \n",
      " 13  kelasrawat   1099338 non-null  int64  \n",
      "dtypes: float64(2), int64(7), object(5)\n",
      "memory usage: 117.4+ MB\n"
     ]
    }
   ],
   "source": [
    "# dataframe info\n",
    "samp.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Proc and Diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ewwwr\\AppData\\Local\\Temp\\ipykernel_1448\\2928535727.py:6: DtypeWarning: Columns (1) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  proc = pd.read_csv(\"temp_data/1/test_proc_cleaned.csv\")\n"
     ]
    }
   ],
   "source": [
    "# load all dataset for samp only using the id and label for merging proc and diag\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "samp = pd.read_csv(\"temp_data/1/test_samp_cleaned.csv\", usecols=[\"id\"])\n",
    "proc = pd.read_csv(\"temp_data/1/test_proc_cleaned.csv\")\n",
    "diag = pd.read_csv(\"temp_data/1/test_diag_cleaned.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>74.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>88.78</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>90.59</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>90.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>90.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858782</th>\n",
       "      <td>1099314</td>\n",
       "      <td>73.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858783</th>\n",
       "      <td>1099318</td>\n",
       "      <td>90.46</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858784</th>\n",
       "      <td>1099318</td>\n",
       "      <td>87.44</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858785</th>\n",
       "      <td>1099322</td>\n",
       "      <td>88.78</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858786</th>\n",
       "      <td>1099337</td>\n",
       "      <td>90.54</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>858787 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id   proc  nproc\n",
       "0             2   74.0    3.0\n",
       "1             2  88.78    3.0\n",
       "2             2  90.59    3.0\n",
       "3             7  90.59    1.0\n",
       "4            10  90.59    1.0\n",
       "...         ...    ...    ...\n",
       "858782  1099314  73.59    1.0\n",
       "858783  1099318  90.46    2.0\n",
       "858784  1099318  87.44    2.0\n",
       "858785  1099322  88.78    1.0\n",
       "858786  1099337  90.54    1.0\n",
       "\n",
       "[858787 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# counting amount of proc on each id\n",
    "# if the calculation is null its actualy 1 caused by .duplicated method only counting\n",
    "# the duplicate value so i fill null value with 1\n",
    "\n",
    "temp = dict(proc[proc[\"id\"].duplicated(keep=False)][\"id\"].value_counts())\n",
    "temp = pd.DataFrame({\n",
    "    \"id\":list(temp.keys()),\n",
    "    \"nproc\":list(temp.values())\n",
    "})\n",
    "proc = pd.merge(proc, temp, on=\"id\", how=\"left\")\n",
    "proc[\"nproc\"].fillna(1, inplace=True)  \n",
    "del temp\n",
    "gc.collect()\n",
    "proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>74.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>90.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>90.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>14</td>\n",
       "      <td>90.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>16</td>\n",
       "      <td>90.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858781</th>\n",
       "      <td>1099308</td>\n",
       "      <td>88.78</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858782</th>\n",
       "      <td>1099314</td>\n",
       "      <td>73.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858783</th>\n",
       "      <td>1099318</td>\n",
       "      <td>90.46</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858785</th>\n",
       "      <td>1099322</td>\n",
       "      <td>88.78</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858786</th>\n",
       "      <td>1099337</td>\n",
       "      <td>90.54</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>560978 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id   proc  nproc\n",
       "0             2   74.0    3.0\n",
       "3             7  90.59    1.0\n",
       "4            10  90.59    1.0\n",
       "5            14  90.59    1.0\n",
       "6            16  90.59    1.0\n",
       "...         ...    ...    ...\n",
       "858781  1099308  88.78    1.0\n",
       "858782  1099314  73.59    1.0\n",
       "858783  1099318  90.46    2.0\n",
       "858785  1099322  88.78    1.0\n",
       "858786  1099337  90.54    1.0\n",
       "\n",
       "[560978 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop the duplicate id because we have got the amount of proc in nproc column\n",
    "proc.drop_duplicates(subset=\"id\", inplace=True)\n",
    "proc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089359</th>\n",
       "      <td>697460</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089360</th>\n",
       "      <td>455269</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089361</th>\n",
       "      <td>1006349</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089362</th>\n",
       "      <td>1093249</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089363</th>\n",
       "      <td>2226</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1089364 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  nproc\n",
       "0           3548    0.0\n",
       "1          85390    0.0\n",
       "2          89574    0.0\n",
       "3          88653    0.0\n",
       "4          88634    0.0\n",
       "...          ...    ...\n",
       "1089359   697460    0.0\n",
       "1089360   455269    0.0\n",
       "1089361  1006349    0.0\n",
       "1089362  1093249    1.0\n",
       "1089363     2226    0.0\n",
       "\n",
       "[1089364 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# left join the nproc column with the samp\n",
    "# fillna with 0 because if there is null it means the patient doesnt have any procedure\n",
    "\n",
    "proc.drop(\"proc\", axis=1, inplace=True)\n",
    "df = pd.merge(samp, proc, on=\"id\", how=\"left\")\n",
    "df[\"nproc\"].fillna(0, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>kddiag</th>\n",
       "      <th>levelid</th>\n",
       "      <th>dkdiag</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>437398</td>\n",
       "      <td>F90.0</td>\n",
       "      <td>2</td>\n",
       "      <td>F90.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>438542</td>\n",
       "      <td>A16.0</td>\n",
       "      <td>2</td>\n",
       "      <td>A16.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>440827</td>\n",
       "      <td>I11.9</td>\n",
       "      <td>2</td>\n",
       "      <td>I11.9</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>446053</td>\n",
       "      <td>I64</td>\n",
       "      <td>2</td>\n",
       "      <td>I64</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>453562</td>\n",
       "      <td>I64</td>\n",
       "      <td>2</td>\n",
       "      <td>I64</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112318</th>\n",
       "      <td>789463</td>\n",
       "      <td>K21.9</td>\n",
       "      <td>2</td>\n",
       "      <td>K21.9</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112319</th>\n",
       "      <td>792932</td>\n",
       "      <td>N18.9</td>\n",
       "      <td>2</td>\n",
       "      <td>N18.9</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112320</th>\n",
       "      <td>794601</td>\n",
       "      <td>H66.9</td>\n",
       "      <td>2</td>\n",
       "      <td>H66.9</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112321</th>\n",
       "      <td>797586</td>\n",
       "      <td>H04.0</td>\n",
       "      <td>2</td>\n",
       "      <td>H04.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112322</th>\n",
       "      <td>798050</td>\n",
       "      <td>N40</td>\n",
       "      <td>2</td>\n",
       "      <td>N40</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1112323 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id kddiag  levelid dkdiag  nsdiag\n",
       "0        437398  F90.0        2  F90.0     1.0\n",
       "1        438542  A16.0        2  A16.0     1.0\n",
       "2        440827  I11.9        2  I11.9     1.0\n",
       "3        446053    I64        2    I64     2.0\n",
       "4        453562    I64        2    I64     1.0\n",
       "...         ...    ...      ...    ...     ...\n",
       "1112318  789463  K21.9        2  K21.9     2.0\n",
       "1112319  792932  N18.9        2  N18.9     1.0\n",
       "1112320  794601  H66.9        2  H66.9     1.0\n",
       "1112321  797586  H04.0        2  H04.0     1.0\n",
       "1112322  798050    N40        2    N40     1.0\n",
       "\n",
       "[1112323 rows x 5 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# same as before counting the secondary diagnose\n",
    "# and fill the null with 1\n",
    "temp = dict(diag[diag[\"id\"].duplicated(keep=False)][\"id\"].value_counts())\n",
    "temp = pd.DataFrame({\n",
    "    \"id\":list(temp.keys()),\n",
    "    \"nsdiag\":list(temp.values())\n",
    "})\n",
    "diag = pd.merge(diag, temp, on=\"id\", how=\"left\")\n",
    "diag[\"nsdiag\"].fillna(1, inplace=True)\n",
    "del temp\n",
    "gc.collect()\n",
    "diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>kddiag</th>\n",
       "      <th>levelid</th>\n",
       "      <th>dkdiag</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>437398</td>\n",
       "      <td>F90.0</td>\n",
       "      <td>2</td>\n",
       "      <td>F90.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>438542</td>\n",
       "      <td>A16.0</td>\n",
       "      <td>2</td>\n",
       "      <td>A16.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>440827</td>\n",
       "      <td>I11.9</td>\n",
       "      <td>2</td>\n",
       "      <td>I11.9</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>446053</td>\n",
       "      <td>I64</td>\n",
       "      <td>2</td>\n",
       "      <td>I64</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>453562</td>\n",
       "      <td>I64</td>\n",
       "      <td>2</td>\n",
       "      <td>I64</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112316</th>\n",
       "      <td>780536</td>\n",
       "      <td>M17.9</td>\n",
       "      <td>2</td>\n",
       "      <td>M17.9</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112319</th>\n",
       "      <td>792932</td>\n",
       "      <td>N18.9</td>\n",
       "      <td>2</td>\n",
       "      <td>N18.9</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112320</th>\n",
       "      <td>794601</td>\n",
       "      <td>H66.9</td>\n",
       "      <td>2</td>\n",
       "      <td>H66.9</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112321</th>\n",
       "      <td>797586</td>\n",
       "      <td>H04.0</td>\n",
       "      <td>2</td>\n",
       "      <td>H04.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112322</th>\n",
       "      <td>798050</td>\n",
       "      <td>N40</td>\n",
       "      <td>2</td>\n",
       "      <td>N40</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>835737 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id kddiag  levelid dkdiag  nsdiag\n",
       "0        437398  F90.0        2  F90.0     1.0\n",
       "1        438542  A16.0        2  A16.0     1.0\n",
       "2        440827  I11.9        2  I11.9     1.0\n",
       "3        446053    I64        2    I64     2.0\n",
       "4        453562    I64        2    I64     1.0\n",
       "...         ...    ...      ...    ...     ...\n",
       "1112316  780536  M17.9        2  M17.9     1.0\n",
       "1112319  792932  N18.9        2  N18.9     1.0\n",
       "1112320  794601  H66.9        2  H66.9     1.0\n",
       "1112321  797586  H04.0        2  H04.0     1.0\n",
       "1112322  798050    N40        2    N40     1.0\n",
       "\n",
       "[835737 rows x 5 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dropping the duplicate id because we have got the nsdiag\n",
    "diag.drop_duplicates(subset=\"id\", inplace=True)\n",
    "diag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089359</th>\n",
       "      <td>697460</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089360</th>\n",
       "      <td>455269</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089361</th>\n",
       "      <td>1006349</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089362</th>\n",
       "      <td>1093249</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089363</th>\n",
       "      <td>2226</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1089364 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  nproc  nsdiag\n",
       "0           3548    0.0     1.0\n",
       "1          85390    0.0     1.0\n",
       "2          89574    0.0     2.0\n",
       "3          88653    0.0     2.0\n",
       "4          88634    0.0     1.0\n",
       "...          ...    ...     ...\n",
       "1089359   697460    0.0     1.0\n",
       "1089360   455269    0.0     0.0\n",
       "1089361  1006349    0.0     1.0\n",
       "1089362  1093249    1.0     0.0\n",
       "1089363     2226    0.0     2.0\n",
       "\n",
       "[1089364 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# left join the nsdiag with other\n",
    "# fillna with 0 because if there is null it means the patient doesnt have any secondary diagnose\n",
    "diag.drop([\"kddiag\", \"levelid\", \"dkdiag\"], axis=1, inplace=True)\n",
    "df = pd.merge(df, diag, on=\"id\", how=\"left\")\n",
    "df[\"nsdiag\"].fillna(0, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop the id column\n",
    "df.drop(\"id\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089359</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089360</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089361</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089362</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089363</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1089364 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         nproc  nsdiag\n",
       "0          0.0     1.0\n",
       "1          0.0     1.0\n",
       "2          0.0     2.0\n",
       "3          0.0     2.0\n",
       "4          0.0     1.0\n",
       "...        ...     ...\n",
       "1089359    0.0     1.0\n",
       "1089360    0.0     0.0\n",
       "1089361    0.0     1.0\n",
       "1089362    1.0     0.0\n",
       "1089363    0.0     2.0\n",
       "\n",
       "[1089364 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# view data\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1089364 entries, 0 to 1089363\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count    Dtype  \n",
      "---  ------  --------------    -----  \n",
      " 0   nproc   1089364 non-null  float64\n",
      " 1   nsdiag  1089364 non-null  float64\n",
      "dtypes: float64(2)\n",
      "memory usage: 24.9 MB\n"
     ]
    }
   ],
   "source": [
    "# dataframe info\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.089364e+06</td>\n",
       "      <td>1.089364e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.602665e-01</td>\n",
       "      <td>1.006093e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.035171e+00</td>\n",
       "      <td>8.037099e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.800000e+01</td>\n",
       "      <td>1.500000e+01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              nproc        nsdiag\n",
       "count  1.089364e+06  1.089364e+06\n",
       "mean   7.602665e-01  1.006093e+00\n",
       "std    1.035171e+00  8.037099e-01\n",
       "min    0.000000e+00  0.000000e+00\n",
       "25%    0.000000e+00  1.000000e+00\n",
       "50%    1.000000e+00  1.000000e+00\n",
       "75%    1.000000e+00  1.000000e+00\n",
       "max    1.800000e+01  1.500000e+01"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dataframe statistic\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Features Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dropping column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>67</td>\n",
       "      <td>L</td>\n",
       "      <td>2020-01-20T17:00:00.000Z</td>\n",
       "      <td>2020-01-20T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>60</td>\n",
       "      <td>L</td>\n",
       "      <td>2019-02-06T17:00:00.000Z</td>\n",
       "      <td>2019-02-06T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>2019-12-11T17:00:00.000Z</td>\n",
       "      <td>2019-12-11T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>52</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-02-24T17:00:00.000Z</td>\n",
       "      <td>2020-02-24T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2020-02-05T17:00:00.000Z</td>\n",
       "      <td>2020-02-05T17:00:00.000Z</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>455269</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2020-12-31T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3457000.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>1006349</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-01T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3416200.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>1093249</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-29T17:00:00.000Z</td>\n",
       "      <td>2021-01-01T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>3762200.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>1003715</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-22T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>9672000.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>2226</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-30T17:00:00.000Z</td>\n",
       "      <td>2021-01-04T17:00:00.000Z</td>\n",
       "      <td>1</td>\n",
       "      <td>1465800.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  usia jenkel                 tgldatang  \\\n",
       "0           3548    67      L  2020-01-20T17:00:00.000Z   \n",
       "1          85390    60      L  2019-02-06T17:00:00.000Z   \n",
       "2          89574    66      P  2019-12-11T17:00:00.000Z   \n",
       "3          88653    52      P  2020-02-24T17:00:00.000Z   \n",
       "4          88634    63      L  2020-02-05T17:00:00.000Z   \n",
       "...          ...   ...    ...                       ...   \n",
       "1099333   455269     0      P  2020-12-29T17:00:00.000Z   \n",
       "1099334  1006349     0      P  2020-12-30T17:00:00.000Z   \n",
       "1099335  1093249     0      P  2020-12-29T17:00:00.000Z   \n",
       "1099336  1003715     0      P  2020-12-30T17:00:00.000Z   \n",
       "1099337     2226     0      P  2020-12-30T17:00:00.000Z   \n",
       "\n",
       "                        tglpulang  jenispel      biaya  jenispulang  \n",
       "0        2020-01-20T17:00:00.000Z         2   187300.0            1  \n",
       "1        2019-02-06T17:00:00.000Z         2   381600.0            1  \n",
       "2        2019-12-11T17:00:00.000Z         2   185500.0            1  \n",
       "3        2020-02-24T17:00:00.000Z         2   185500.0            1  \n",
       "4        2020-02-05T17:00:00.000Z         2   185500.0            1  \n",
       "...                           ...       ...        ...          ...  \n",
       "1099333  2020-12-31T17:00:00.000Z         1  3457000.0            1  \n",
       "1099334  2021-01-01T17:00:00.000Z         1  3416200.0            1  \n",
       "1099335  2021-01-01T17:00:00.000Z         1  3762200.0            1  \n",
       "1099336  2021-01-22T17:00:00.000Z         1  9672000.0            1  \n",
       "1099337  2021-01-04T17:00:00.000Z         1  1465800.0            1  \n",
       "\n",
       "[1099338 rows x 8 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dropping unused column based on the heatmap and pvalue in eda phase\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "samp = pd.read_csv(\"temp_data/1/test_samp_cleaned.csv\")\n",
    "samp.drop([\"id_peserta\", \"pisat\", \"kelasrawat\", \"dati2\", \"typefaskes\", \"cbg\"], axis=1, inplace=True)\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save to some folder\n",
    "samp.to_csv(\"temp_data/1/test_samp_cleaned.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handle nday"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>tgldatang</th>\n",
       "      <th>tglpulang</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>nday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>67</td>\n",
       "      <td>L</td>\n",
       "      <td>2020-01-20 17:00:00+00:00</td>\n",
       "      <td>2020-01-20 17:00:00+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>60</td>\n",
       "      <td>L</td>\n",
       "      <td>2019-02-06 17:00:00+00:00</td>\n",
       "      <td>2019-02-06 17:00:00+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>2019-12-11 17:00:00+00:00</td>\n",
       "      <td>2019-12-11 17:00:00+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>52</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-02-24 17:00:00+00:00</td>\n",
       "      <td>2020-02-24 17:00:00+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2020-02-05 17:00:00+00:00</td>\n",
       "      <td>2020-02-05 17:00:00+00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>455269</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-29 17:00:00+00:00</td>\n",
       "      <td>2020-12-31 17:00:00+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>3457000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>1006349</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-30 17:00:00+00:00</td>\n",
       "      <td>2021-01-01 17:00:00+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>3416200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>1093249</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-29 17:00:00+00:00</td>\n",
       "      <td>2021-01-01 17:00:00+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>3762200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>1003715</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-30 17:00:00+00:00</td>\n",
       "      <td>2021-01-22 17:00:00+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>9672000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>2226</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>2020-12-30 17:00:00+00:00</td>\n",
       "      <td>2021-01-04 17:00:00+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>1465800.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  usia jenkel                 tgldatang  \\\n",
       "0           3548    67      L 2020-01-20 17:00:00+00:00   \n",
       "1          85390    60      L 2019-02-06 17:00:00+00:00   \n",
       "2          89574    66      P 2019-12-11 17:00:00+00:00   \n",
       "3          88653    52      P 2020-02-24 17:00:00+00:00   \n",
       "4          88634    63      L 2020-02-05 17:00:00+00:00   \n",
       "...          ...   ...    ...                       ...   \n",
       "1099333   455269     0      P 2020-12-29 17:00:00+00:00   \n",
       "1099334  1006349     0      P 2020-12-30 17:00:00+00:00   \n",
       "1099335  1093249     0      P 2020-12-29 17:00:00+00:00   \n",
       "1099336  1003715     0      P 2020-12-30 17:00:00+00:00   \n",
       "1099337     2226     0      P 2020-12-30 17:00:00+00:00   \n",
       "\n",
       "                        tglpulang  jenispel      biaya  jenispulang  nday  \n",
       "0       2020-01-20 17:00:00+00:00         2   187300.0            1     0  \n",
       "1       2019-02-06 17:00:00+00:00         2   381600.0            1     0  \n",
       "2       2019-12-11 17:00:00+00:00         2   185500.0            1     0  \n",
       "3       2020-02-24 17:00:00+00:00         2   185500.0            1     0  \n",
       "4       2020-02-05 17:00:00+00:00         2   185500.0            1     0  \n",
       "...                           ...       ...        ...          ...   ...  \n",
       "1099333 2020-12-31 17:00:00+00:00         1  3457000.0            1     2  \n",
       "1099334 2021-01-01 17:00:00+00:00         1  3416200.0            1     2  \n",
       "1099335 2021-01-01 17:00:00+00:00         1  3762200.0            1     3  \n",
       "1099336 2021-01-22 17:00:00+00:00         1  9672000.0            1    23  \n",
       "1099337 2021-01-04 17:00:00+00:00         1  1465800.0            1     5  \n",
       "\n",
       "[1099338 rows x 9 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make nday column based on tglpulang-tgldatang in days\n",
    "\n",
    "import pandas as pd\n",
    "df = pd.read_csv(\"temp_data/1/test_samp_cleaned.csv\")\n",
    "df[\"tgldatang\"] = pd.to_datetime(df[\"tgldatang\"], format=\"%Y/%m/%dT%H:%M:%S\")\n",
    "df[\"tglpulang\"] = pd.to_datetime(df[\"tglpulang\"], format=\"%Y/%m/%dT%H:%M:%S\")\n",
    "df[\"nday\"] = df[\"tglpulang\"] - df[\"tgldatang\"]\n",
    "df[\"nday\"] = pd.to_timedelta(df[\"nday\"]).dt.days\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>nday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>67</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>60</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>52</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>455269</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>3457000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>1006349</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>3416200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>1093249</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>3762200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>1003715</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>9672000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>2226</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>1465800.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  usia jenkel  jenispel      biaya  jenispulang  nday\n",
       "0           3548    67      L         2   187300.0            1     0\n",
       "1          85390    60      L         2   381600.0            1     0\n",
       "2          89574    66      P         2   185500.0            1     0\n",
       "3          88653    52      P         2   185500.0            1     0\n",
       "4          88634    63      L         2   185500.0            1     0\n",
       "...          ...   ...    ...       ...        ...          ...   ...\n",
       "1099333   455269     0      P         1  3457000.0            1     2\n",
       "1099334  1006349     0      P         1  3416200.0            1     2\n",
       "1099335  1093249     0      P         1  3762200.0            1     3\n",
       "1099336  1003715     0      P         1  9672000.0            1    23\n",
       "1099337     2226     0      P         1  1465800.0            1     5\n",
       "\n",
       "[1099338 rows x 7 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# droping tglpulang tgldatang\n",
    "df.drop([\"tgldatang\", \"tglpulang\"], axis=1, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save to some folder\n",
    "df.to_csv(\"temp_data/1/ready_test_samp.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handle nproc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ewwwr\\AppData\\Local\\Temp\\ipykernel_10232\\2248940628.py:6: DtypeWarning: Columns (1) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(\"temp_data/1/test_proc_cleaned.csv\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>74.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>90.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858782</th>\n",
       "      <td>1099314</td>\n",
       "      <td>73.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858783</th>\n",
       "      <td>1099318</td>\n",
       "      <td>90.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858784</th>\n",
       "      <td>1099318</td>\n",
       "      <td>87.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858785</th>\n",
       "      <td>1099322</td>\n",
       "      <td>88.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858786</th>\n",
       "      <td>1099337</td>\n",
       "      <td>90.54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>858787 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id   proc\n",
       "0             2   74.0\n",
       "1             2  88.78\n",
       "2             2  90.59\n",
       "3             7  90.59\n",
       "4            10  90.59\n",
       "...         ...    ...\n",
       "858782  1099314  73.59\n",
       "858783  1099318  90.46\n",
       "858784  1099318  87.44\n",
       "858785  1099322  88.78\n",
       "858786  1099337  90.54\n",
       "\n",
       "[858787 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the procedure dataset is counted how many procedure each id have and save in nproc column\n",
    "\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "df = pd.read_csv(\"temp_data/1/test_proc_cleaned.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>proc</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>74.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>88.78</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>90.59</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>90.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>90.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858782</th>\n",
       "      <td>1099314</td>\n",
       "      <td>73.59</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858783</th>\n",
       "      <td>1099318</td>\n",
       "      <td>90.46</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858784</th>\n",
       "      <td>1099318</td>\n",
       "      <td>87.44</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858785</th>\n",
       "      <td>1099322</td>\n",
       "      <td>88.78</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858786</th>\n",
       "      <td>1099337</td>\n",
       "      <td>90.54</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>858787 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id   proc  nproc\n",
       "0             2   74.0    3.0\n",
       "1             2  88.78    3.0\n",
       "2             2  90.59    3.0\n",
       "3             7  90.59    1.0\n",
       "4            10  90.59    1.0\n",
       "...         ...    ...    ...\n",
       "858782  1099314  73.59    1.0\n",
       "858783  1099318  90.46    2.0\n",
       "858784  1099318  87.44    2.0\n",
       "858785  1099322  88.78    1.0\n",
       "858786  1099337  90.54    1.0\n",
       "\n",
       "[858787 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# counting the duplicate id for nproc\n",
    "# same as in the eda phase\n",
    "\n",
    "temp = dict(df[df[\"id\"].duplicated(keep=False)][\"id\"].value_counts())\n",
    "temp = pd.DataFrame({\n",
    "    \"id\":list(temp.keys()),\n",
    "    \"nproc\":list(temp.values())\n",
    "})\n",
    "df = pd.merge(df, temp, on=\"id\", how=\"left\")\n",
    "df[\"nproc\"].fillna(1, inplace=True)\n",
    "del temp\n",
    "gc.collect()\n",
    "# print(1)\n",
    "\n",
    "# # one hot encoding\n",
    "# encoded_proc = pd.get_dummies(df[\"proc\"])\n",
    "# encoded_proc.drop([2424], axis=1, inplace=True)\n",
    "# df.drop([\"proc\"], axis=1, inplace=True)\n",
    "# print(2)\n",
    "\n",
    "# # just in case if there is incomplete procedure in dataset eg only 0-50 not 0-99\n",
    "# # so im creating mechanism that if thereis only 0-50 im creating the 51-99 column\n",
    "# ls_proc = [x for x in range(100)]\n",
    "# ls_columns = list(encoded_proc.columns)\n",
    "# temp = {}\n",
    "# for i in ls_proc:\n",
    "#     if i not in ls_columns:\n",
    "#         temp.__setitem__(i, [None])\n",
    "# print(3)\n",
    "\n",
    "# # concat the created column to encoded procedure\n",
    "# temp = pd.DataFrame(temp)\n",
    "# encoded_proc = pd.concat([encoded_proc, temp], axis=1)\n",
    "# encoded_proc.sort_index(axis=1, inplace=True)\n",
    "# encoded_proc.fillna(0, inplace=True)\n",
    "# print(4)\n",
    "\n",
    "# # concat encoded procedure to dataframe\n",
    "# df = pd.concat([df, encoded_proc], axis=1)\n",
    "# del encoded_proc, temp\n",
    "# gc.collect()\n",
    "# print(5)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id       0\n",
       "proc     0\n",
       "nproc    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check null\n",
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>14</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>16</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858781</th>\n",
       "      <td>1099308</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858782</th>\n",
       "      <td>1099314</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858783</th>\n",
       "      <td>1099318</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858785</th>\n",
       "      <td>1099322</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858786</th>\n",
       "      <td>1099337</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>560978 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  nproc\n",
       "0             2    3.0\n",
       "3             7    1.0\n",
       "4            10    1.0\n",
       "5            14    1.0\n",
       "6            16    1.0\n",
       "...         ...    ...\n",
       "858781  1099308    1.0\n",
       "858782  1099314    1.0\n",
       "858783  1099318    2.0\n",
       "858785  1099322    1.0\n",
       "858786  1099337    1.0\n",
       "\n",
       "[560978 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop proc column and drop duplicate\n",
    "df.drop(\"proc\", axis=1, inplace=True)\n",
    "df.drop_duplicates(subset=\"id\", inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>nproc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>14</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>16</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858781</th>\n",
       "      <td>1099308</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858782</th>\n",
       "      <td>1099314</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858783</th>\n",
       "      <td>1099318</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858785</th>\n",
       "      <td>1099322</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858786</th>\n",
       "      <td>1099337</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>560978 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  nproc\n",
       "0             2    3.0\n",
       "3             7    1.0\n",
       "4            10    1.0\n",
       "5            14    1.0\n",
       "6            16    1.0\n",
       "...         ...    ...\n",
       "858781  1099308    1.0\n",
       "858782  1099314    1.0\n",
       "858783  1099318    2.0\n",
       "858785  1099322    1.0\n",
       "858786  1099337    1.0\n",
       "\n",
       "[560978 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save to some folder\n",
    "df.to_csv(\"temp_data/1/ready_test_proc.csv\", index=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handle nsdiag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>kddiag</th>\n",
       "      <th>levelid</th>\n",
       "      <th>dkdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>437398</td>\n",
       "      <td>F90.0</td>\n",
       "      <td>2</td>\n",
       "      <td>F90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>438542</td>\n",
       "      <td>A16.0</td>\n",
       "      <td>2</td>\n",
       "      <td>A16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>440827</td>\n",
       "      <td>I11.9</td>\n",
       "      <td>2</td>\n",
       "      <td>I11.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>446053</td>\n",
       "      <td>I64</td>\n",
       "      <td>2</td>\n",
       "      <td>I64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>453562</td>\n",
       "      <td>I64</td>\n",
       "      <td>2</td>\n",
       "      <td>I64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112318</th>\n",
       "      <td>789463</td>\n",
       "      <td>K21.9</td>\n",
       "      <td>2</td>\n",
       "      <td>K21.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112319</th>\n",
       "      <td>792932</td>\n",
       "      <td>N18.9</td>\n",
       "      <td>2</td>\n",
       "      <td>N18.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112320</th>\n",
       "      <td>794601</td>\n",
       "      <td>H66.9</td>\n",
       "      <td>2</td>\n",
       "      <td>H66.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112321</th>\n",
       "      <td>797586</td>\n",
       "      <td>H04.0</td>\n",
       "      <td>2</td>\n",
       "      <td>H04.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112322</th>\n",
       "      <td>798050</td>\n",
       "      <td>N40</td>\n",
       "      <td>2</td>\n",
       "      <td>N40</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1112323 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id kddiag  levelid dkdiag\n",
       "0        437398  F90.0        2  F90.0\n",
       "1        438542  A16.0        2  A16.0\n",
       "2        440827  I11.9        2  I11.9\n",
       "3        446053    I64        2    I64\n",
       "4        453562    I64        2    I64\n",
       "...         ...    ...      ...    ...\n",
       "1112318  789463  K21.9        2  K21.9\n",
       "1112319  792932  N18.9        2  N18.9\n",
       "1112320  794601  H66.9        2  H66.9\n",
       "1112321  797586  H04.0        2  H04.0\n",
       "1112322  798050    N40        2    N40\n",
       "\n",
       "[1112323 rows x 4 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading data\n",
    "\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "df = pd.read_csv(\"temp_data/1/test_diag_cleaned.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>kddiag</th>\n",
       "      <th>levelid</th>\n",
       "      <th>dkdiag</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>437398</td>\n",
       "      <td>F90.0</td>\n",
       "      <td>2</td>\n",
       "      <td>F90.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>438542</td>\n",
       "      <td>A16.0</td>\n",
       "      <td>2</td>\n",
       "      <td>A16.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>440827</td>\n",
       "      <td>I11.9</td>\n",
       "      <td>2</td>\n",
       "      <td>I11.9</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>446053</td>\n",
       "      <td>I64</td>\n",
       "      <td>2</td>\n",
       "      <td>I64</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>453562</td>\n",
       "      <td>I64</td>\n",
       "      <td>2</td>\n",
       "      <td>I64</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112318</th>\n",
       "      <td>789463</td>\n",
       "      <td>K21.9</td>\n",
       "      <td>2</td>\n",
       "      <td>K21.9</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112319</th>\n",
       "      <td>792932</td>\n",
       "      <td>N18.9</td>\n",
       "      <td>2</td>\n",
       "      <td>N18.9</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112320</th>\n",
       "      <td>794601</td>\n",
       "      <td>H66.9</td>\n",
       "      <td>2</td>\n",
       "      <td>H66.9</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112321</th>\n",
       "      <td>797586</td>\n",
       "      <td>H04.0</td>\n",
       "      <td>2</td>\n",
       "      <td>H04.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112322</th>\n",
       "      <td>798050</td>\n",
       "      <td>N40</td>\n",
       "      <td>2</td>\n",
       "      <td>N40</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1112323 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id kddiag  levelid dkdiag  nsdiag\n",
       "0        437398  F90.0        2  F90.0     1.0\n",
       "1        438542  A16.0        2  A16.0     1.0\n",
       "2        440827  I11.9        2  I11.9     1.0\n",
       "3        446053    I64        2    I64     2.0\n",
       "4        453562    I64        2    I64     1.0\n",
       "...         ...    ...      ...    ...     ...\n",
       "1112318  789463  K21.9        2  K21.9     2.0\n",
       "1112319  792932  N18.9        2  N18.9     1.0\n",
       "1112320  794601  H66.9        2  H66.9     1.0\n",
       "1112321  797586  H04.0        2  H04.0     1.0\n",
       "1112322  798050    N40        2    N40     1.0\n",
       "\n",
       "[1112323 rows x 5 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# counting the duplicate id for secondary diagnose as nsdiag value\n",
    "# same as in the eda phase\n",
    "temp = dict(df[df[\"id\"].duplicated(keep=False)][\"id\"].value_counts())\n",
    "temp = pd.DataFrame({\n",
    "    \"id\":list(temp.keys()),\n",
    "    \"nsdiag\":list(temp.values())\n",
    "})\n",
    "df = pd.merge(df, temp, on=\"id\", how=\"left\")\n",
    "df[\"nsdiag\"].fillna(1, inplace=True)\n",
    "del temp\n",
    "gc.collect()\n",
    "# print(1)\n",
    "\n",
    "# # one hot encoding\n",
    "# encoded_proc = pd.get_dummies(df[\"proc\"])\n",
    "# encoded_proc.drop([2424], axis=1, inplace=True)\n",
    "# df.drop([\"proc\"], axis=1, inplace=True)\n",
    "# print(2)\n",
    "\n",
    "# # just in case if there is incomplete procedure in dataset eg only 0-50 not 0-99\n",
    "# # so im creating mechanism that if thereis only 0-50 im creating the 51-99 column\n",
    "# ls_proc = [x for x in range(100)]\n",
    "# ls_columns = list(encoded_proc.columns)\n",
    "# temp = {}\n",
    "# for i in ls_proc:\n",
    "#     if i not in ls_columns:\n",
    "#         temp.__setitem__(i, [None])\n",
    "# print(3)\n",
    "\n",
    "# # concat the created column to encoded procedure\n",
    "# temp = pd.DataFrame(temp)\n",
    "# encoded_proc = pd.concat([encoded_proc, temp], axis=1)\n",
    "# encoded_proc.sort_index(axis=1, inplace=True)\n",
    "# encoded_proc.fillna(0, inplace=True)\n",
    "# print(4)\n",
    "\n",
    "# # concat encoded procedure to dataframe\n",
    "# df = pd.concat([df, encoded_proc], axis=1)\n",
    "# del encoded_proc, temp\n",
    "# gc.collect()\n",
    "# print(5)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id         0\n",
       "kddiag     0\n",
       "levelid    0\n",
       "dkdiag     0\n",
       "nsdiag     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check null\n",
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>437398</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>438542</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>440827</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>446053</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>453562</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112316</th>\n",
       "      <td>780536</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112319</th>\n",
       "      <td>792932</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112320</th>\n",
       "      <td>794601</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112321</th>\n",
       "      <td>797586</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112322</th>\n",
       "      <td>798050</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>835737 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  nsdiag\n",
       "0        437398     1.0\n",
       "1        438542     1.0\n",
       "2        440827     1.0\n",
       "3        446053     2.0\n",
       "4        453562     1.0\n",
       "...         ...     ...\n",
       "1112316  780536     1.0\n",
       "1112319  792932     1.0\n",
       "1112320  794601     1.0\n",
       "1112321  797586     1.0\n",
       "1112322  798050     1.0\n",
       "\n",
       "[835737 rows x 2 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop proc column and drop duplicate\n",
    "df.drop([\"kddiag\", \"dkdiag\", \"levelid\"], axis=1, inplace=True)\n",
    "df.drop_duplicates(subset=\"id\", inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>437398</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>438542</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>440827</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>446053</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>453562</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112316</th>\n",
       "      <td>780536</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112319</th>\n",
       "      <td>792932</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112320</th>\n",
       "      <td>794601</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112321</th>\n",
       "      <td>797586</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112322</th>\n",
       "      <td>798050</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>835737 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  nsdiag\n",
       "0        437398     1.0\n",
       "1        438542     1.0\n",
       "2        440827     1.0\n",
       "3        446053     2.0\n",
       "4        453562     1.0\n",
       "...         ...     ...\n",
       "1112316  780536     1.0\n",
       "1112319  792932     1.0\n",
       "1112320  794601     1.0\n",
       "1112321  797586     1.0\n",
       "1112322  798050     1.0\n",
       "\n",
       "[835737 rows x 2 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save to some folder\n",
    "df.to_csv(\"temp_data/1/ready_test_diag.csv\", index=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoding Categorical Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for categorical data im encode it and one hot enocode it but im only creating a function here\n",
    "# this function called from somewhere soon because for ram reason im not encoding all 11M+ data\n",
    "# only data that il use in train and test to encode\n",
    "\n",
    "# after one hot encoding im dropping one column because its for efficiency\n",
    "# example there is sex data that have value P and L after one hot encoding it will be have P colum and L colum\n",
    "# if male it will be 1 for P and 0 for L, actualy it can be described only one of those column\n",
    "# so if i drop no matter what column eg the L column, if its male the P is 0 and female P is 1\n",
    "\n",
    "from pandas import DataFrame\n",
    "import numpy as np\n",
    "import gc\n",
    "\n",
    "def encode_categorical(df:DataFrame):\n",
    "\n",
    "    try:\n",
    "        df[\"id\"] = df[\"id\"].astype(np.int64)\n",
    "    except:\n",
    "        print(\"no id column skip formating for id column to np.int64\")\n",
    "    try:\n",
    "        df[\"typefaskes\"] = df[\"typefaskes\"].astype(str)\n",
    "    except:\n",
    "        print(\"no typefaskes column skip formating for typefaskes column to str\")\n",
    "    try:\n",
    "        df[\"kelasrawat\"] = df[\"kelasrawat\"].astype(np.int8)\n",
    "    except:\n",
    "        print(\"no kelasrawat column skip formating for kelasrawat column to str\")\n",
    "\n",
    "    df[\"usia\"] = df[\"usia\"].astype(np.int8)\n",
    "    df[\"jenkel\"] = df[\"jenkel\"].astype(str)\n",
    "    df[\"jenispel\"] = df[\"jenispel\"].astype(str)\n",
    "    df[\"jenispulang\"] = df[\"jenispulang\"].astype(str)\n",
    "    df[\"biaya\"] = df[\"biaya\"].astype(float)\n",
    "\n",
    "    try:\n",
    "        df[\"label\"] = df[\"label\"].astype(np.int8)\n",
    "    except:\n",
    "        print(\"no label column skip formating for label column to np.int8\")\n",
    "\n",
    "    try:\n",
    "        encoded_typefaskes = pd.get_dummies(df[\"typefaskes\"])\n",
    "        try:\n",
    "            encoded_typefaskes.drop([\"KK\"], axis=1, inplace=True)\n",
    "        except:\n",
    "            print(\"no KK column skip creating KK column\")\n",
    "        df.drop([\"typefaskes\"], axis=1, inplace=True)\n",
    "        df = pd.concat([df, encoded_typefaskes], axis=1)\n",
    "        del encoded_typefaskes\n",
    "        gc.collect()\n",
    "    except:\n",
    "        print(\"no typefaskes column skip encoding typefaskes\")\n",
    "\n",
    "    encoded_jenkel = pd.get_dummies(df[\"jenkel\"])\n",
    "    encoded_jenkel.drop([\"P\"], axis=1, inplace=True)\n",
    "    encoded_jenkel = encoded_jenkel.rename(columns={\"L\":\"ismale\"})\n",
    "    df.drop([\"jenkel\"], axis=1, inplace=True)\n",
    "    df = pd.concat([df, encoded_jenkel], axis=1)\n",
    "    del encoded_jenkel\n",
    "    gc.collect()\n",
    "\n",
    "    encoded_jenispel = pd.get_dummies(df[\"jenispel\"])\n",
    "    encoded_jenispel.drop([\"2\"], axis=1, inplace=True)\n",
    "    encoded_jenispel = encoded_jenispel.rename(columns={\"1\":\"jenpel1\"})\n",
    "    df.drop([\"jenispel\"], axis=1, inplace=True)\n",
    "    df = pd.concat([df, encoded_jenispel], axis=1)\n",
    "    del encoded_jenispel\n",
    "    gc.collect()\n",
    "\n",
    "    encoded_jenispulang = pd.get_dummies(df[\"jenispulang\"])\n",
    "    encoded_jenispulang.drop([\"5.0\"], axis=1, inplace=True)\n",
    "    encoded_jenispulang = encoded_jenispulang.rename(columns={\n",
    "                                                    \"1.0\":\"jenpul1\",\n",
    "                                                    \"2.0\":\"jenpul2\",\n",
    "                                                    \"3.0\":\"jenpul3\",\n",
    "                                                    \"4.0\":\"jenpul4\",\n",
    "                                                })\n",
    "    df.drop([\"jenispulang\"], axis=1, inplace=True)\n",
    "    df = pd.concat([df, encoded_jenispulang], axis=1)\n",
    "    del encoded_jenispulang\n",
    "    gc.collect()\n",
    "\n",
    "    # encoded_proc = pd.get_dummies(df[\"proc\"])\n",
    "    # encoded_proc.drop([\"2424\"], axis=1, inplace=True)\n",
    "    # df.drop([\"proc\"], axis=1, inplace=True)\n",
    "    # df = pd.concat([df, encoded_proc], axis=1)\n",
    "    # del encoded_proc\n",
    "    # gc.collect()\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>nday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>67</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>60</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>52</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>455269</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>3457000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>1006349</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>3416200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>1093249</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>3762200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>1003715</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>9672000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>2226</td>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "      <td>1</td>\n",
       "      <td>1465800.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  usia jenkel  jenispel      biaya  jenispulang  nday\n",
       "0           3548    67      L         2   187300.0            1     0\n",
       "1          85390    60      L         2   381600.0            1     0\n",
       "2          89574    66      P         2   185500.0            1     0\n",
       "3          88653    52      P         2   185500.0            1     0\n",
       "4          88634    63      L         2   185500.0            1     0\n",
       "...          ...   ...    ...       ...        ...          ...   ...\n",
       "1099333   455269     0      P         1  3457000.0            1     2\n",
       "1099334  1006349     0      P         1  3416200.0            1     2\n",
       "1099335  1093249     0      P         1  3762200.0            1     3\n",
       "1099336  1003715     0      P         1  9672000.0            1    23\n",
       "1099337     2226     0      P         1  1465800.0            1     5\n",
       "\n",
       "[1099338 rows x 7 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read all data\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "samp = pd.read_csv(\"temp_data/1/ready_test_samp.csv\")\n",
    "proc = pd.read_csv(\"temp_data/1/ready_test_proc.csv\")\n",
    "diag = pd.read_csv(\"temp_data/1/ready_test_diag.csv\")\n",
    "samp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "usia           0\n",
       "jenkel         0\n",
       "jenispel       0\n",
       "biaya          0\n",
       "jenispulang    0\n",
       "nday           0\n",
       "nproc          0\n",
       "nsdiag         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# left join data\n",
    "df = pd.merge(samp, proc, on=\"id\", how=\"left\")\n",
    "df = pd.merge(df, diag, on=\"id\", how=\"left\")\n",
    "df[\"nproc\"].fillna(0, inplace=True)\n",
    "df[\"nsdiag\"].fillna(0, inplace=True)\n",
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>jenkel</th>\n",
       "      <th>jenispel</th>\n",
       "      <th>biaya</th>\n",
       "      <th>jenispulang</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>67</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>60</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>66</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>52</td>\n",
       "      <td>P</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>63</td>\n",
       "      <td>L</td>\n",
       "      <td>2</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id  usia jenkel  jenispel     biaya  jenispulang  nday  nproc  nsdiag\n",
       "0   3548    67      L         2  187300.0            1     0    0.0     1.0\n",
       "1  85390    60      L         2  381600.0            1     0    0.0     1.0\n",
       "2  89574    66      P         2  185500.0            1     0    0.0     2.0\n",
       "3  88653    52      P         2  185500.0            1     0    0.0     2.0\n",
       "4  88634    63      L         2  185500.0            1     0    0.0     1.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check df\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# jenispulang to float for matching with encode_categorical feature 1.0, 2.0, etc\n",
    "df[\"jenispulang\"] = df[\"jenispulang\"].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no typefaskes column skip formating for typefaskes column to str\n",
      "no kelasrawat column skip formating for kelasrawat column to str\n",
      "no label column skip formating for label column to np.int8\n",
      "no typefaskes column skip encoding typefaskes\n"
     ]
    }
   ],
   "source": [
    "# encode df\n",
    "df = encode_categorical(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>usia</th>\n",
       "      <th>biaya</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>ismale</th>\n",
       "      <th>jenpel1</th>\n",
       "      <th>jenpul1</th>\n",
       "      <th>jenpul2</th>\n",
       "      <th>jenpul3</th>\n",
       "      <th>jenpul4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>67</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>60</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>66</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>52</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>63</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id  usia     biaya  nday  nproc  nsdiag  ismale  jenpel1  jenpul1  \\\n",
       "0   3548    67  187300.0     0    0.0     1.0       1        0        1   \n",
       "1  85390    60  381600.0     0    0.0     1.0       1        0        1   \n",
       "2  89574    66  185500.0     0    0.0     2.0       0        0        1   \n",
       "3  88653    52  185500.0     0    0.0     2.0       0        0        1   \n",
       "4  88634    63  185500.0     0    0.0     1.0       1        0        1   \n",
       "\n",
       "   jenpul2  jenpul3  jenpul4  \n",
       "0        0        0        0  \n",
       "1        0        0        0  \n",
       "2        0        0        0  \n",
       "3        0        0        0  \n",
       "4        0        0        0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check df\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop id column\n",
    "df.drop(\"id\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>usia</th>\n",
       "      <th>biaya</th>\n",
       "      <th>nday</th>\n",
       "      <th>nproc</th>\n",
       "      <th>nsdiag</th>\n",
       "      <th>ismale</th>\n",
       "      <th>jenpel1</th>\n",
       "      <th>jenpul1</th>\n",
       "      <th>jenpul2</th>\n",
       "      <th>jenpul3</th>\n",
       "      <th>jenpul4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.614679</td>\n",
       "      <td>187300.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.641622e-08</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.550459</td>\n",
       "      <td>381600.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.641622e-08</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.605505</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.128324e-07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.477064</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.128324e-07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.577982</td>\n",
       "      <td>185500.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.641622e-08</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3457000.0</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3416200.0</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.641622e-08</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3762200.0</td>\n",
       "      <td>0.004386</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>9672000.0</td>\n",
       "      <td>0.033626</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>1.692486e-07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1465800.0</td>\n",
       "      <td>0.007310</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.128324e-07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             usia      biaya      nday     nproc        nsdiag  ismale  \\\n",
       "0        0.614679   187300.0  0.000000  0.000000  5.641622e-08     1.0   \n",
       "1        0.550459   381600.0  0.000000  0.000000  5.641622e-08     1.0   \n",
       "2        0.605505   185500.0  0.000000  0.000000  1.128324e-07     0.0   \n",
       "3        0.477064   185500.0  0.000000  0.000000  1.128324e-07     0.0   \n",
       "4        0.577982   185500.0  0.000000  0.000000  5.641622e-08     1.0   \n",
       "...           ...        ...       ...       ...           ...     ...   \n",
       "1099333  0.000000  3457000.0  0.002924  0.000000  0.000000e+00     0.0   \n",
       "1099334  0.000000  3416200.0  0.002924  0.000000  5.641622e-08     0.0   \n",
       "1099335  0.000000  3762200.0  0.004386  0.045455  0.000000e+00     0.0   \n",
       "1099336  0.000000  9672000.0  0.033626  0.090909  1.692486e-07     0.0   \n",
       "1099337  0.000000  1465800.0  0.007310  0.000000  1.128324e-07     0.0   \n",
       "\n",
       "         jenpel1  jenpul1  jenpul2  jenpul3  jenpul4  \n",
       "0            0.0      1.0      0.0      0.0      0.0  \n",
       "1            0.0      1.0      0.0      0.0      0.0  \n",
       "2            0.0      1.0      0.0      0.0      0.0  \n",
       "3            0.0      1.0      0.0      0.0      0.0  \n",
       "4            0.0      1.0      0.0      0.0      0.0  \n",
       "...          ...      ...      ...      ...      ...  \n",
       "1099333      1.0      1.0      0.0      0.0      0.0  \n",
       "1099334      1.0      1.0      0.0      0.0      0.0  \n",
       "1099335      1.0      1.0      0.0      0.0      0.0  \n",
       "1099336      1.0      1.0      0.0      0.0      0.0  \n",
       "1099337      1.0      1.0      0.0      0.0      0.0  \n",
       "\n",
       "[1099338 rows x 11 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# using minmax scaler to scale all column\n",
    "# save the fitted scaler to some folder, it usefull for reused in future implementation\n",
    "\n",
    "import joblib\n",
    "\n",
    "bpjs_mmscaler = joblib.load(\"bpjs_mmcaler.save\")\n",
    "columns = df.drop([\"biaya\"], axis=1).columns\n",
    "\n",
    "df[columns] = bpjs_mmscaler.transform(df[columns])\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import library\n",
    "from keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34355/34355 [==============================] - 39s 1ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.],\n",
       "       [0.],\n",
       "       [0.],\n",
       "       ...,\n",
       "       [0.],\n",
       "       [0.],\n",
       "       [0.]], dtype=float32)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load model and start predicting\n",
    "model = load_model(\"model/nn_v2.h5\")\n",
    "ypred = model.predict(df)\n",
    "ypred = np.round(ypred)\n",
    "ypred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1099338"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check len\n",
    "len(ypred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         label\n",
       "0          0.0\n",
       "1          0.0\n",
       "2          0.0\n",
       "3          0.0\n",
       "4          0.0\n",
       "...        ...\n",
       "1099333    0.0\n",
       "1099334    0.0\n",
       "1099335    0.0\n",
       "1099336    0.0\n",
       "1099337    0.0\n",
       "\n",
       "[1099338 rows x 1 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set ypred to dataframe\n",
    "ypred = pd.DataFrame({\n",
    "    \"label\":ypred.ravel()\n",
    "})\n",
    "ypred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>455269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>1006349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>1093249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>1003715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>2226</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id\n",
       "0           3548\n",
       "1          85390\n",
       "2          89574\n",
       "3          88653\n",
       "4          88634\n",
       "...          ...\n",
       "1099333   455269\n",
       "1099334  1006349\n",
       "1099335  1093249\n",
       "1099336  1003715\n",
       "1099337     2226\n",
       "\n",
       "[1099338 rows x 1 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load id so match with leaderboard format\n",
    "id = pd.read_csv(\"dataset/test/sampling3_healthkathon2022_sep.csv\", sep=\";\", usecols=[\"id\"])\n",
    "id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3548</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85390</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89574</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88653</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>88634</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099333</th>\n",
       "      <td>455269</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099334</th>\n",
       "      <td>1006349</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099335</th>\n",
       "      <td>1093249</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099336</th>\n",
       "      <td>1003715</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099337</th>\n",
       "      <td>2226</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1099338 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  label\n",
       "0           3548    0.0\n",
       "1          85390    0.0\n",
       "2          89574    0.0\n",
       "3          88653    0.0\n",
       "4          88634    0.0\n",
       "...          ...    ...\n",
       "1099333   455269    0.0\n",
       "1099334  1006349    0.0\n",
       "1099335  1093249    0.0\n",
       "1099336  1003715    0.0\n",
       "1099337     2226    0.0\n",
       "\n",
       "[1099338 rows x 2 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concat id and label\n",
    "id = pd.concat([id, ypred], axis=1)\n",
    "id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save to some folder\n",
    "id.to_csv(\"answer.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.7 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "c71b0acf67dc15497311fa72cf631c1d9d1cfd28e81044e76d7ee61907bfdeb0"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}