train-from-file-generic-clean-pixel-custom-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.274350Z",
     "start_time": "2020-09-15T07:10:05.162843Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext nb_black\n",
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "# from tensorflow.keras.applications import (\n",
    "#     MobileNetV2,\n",
    "# )\n",
    "from efficientnet.tfkeras import EfficientNetB0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.276296Z",
     "start_time": "2020-09-15T07:10:05.164Z"
    }
   },
   "outputs": [],
   "source": [
    "# dataset_name = \"santander-customer-satisfaction\"\n",
    "# IMAGE_SIZE = 640\n",
    "# dataset_name = \"springleaf-marketing-response\"\n",
    "# IMAGE_SIZE = 1408\n",
    "# dataset_name = \"segment\"\n",
    "# IMAGE_SIZE = 160\n",
    "# dataset_name = \"rl\"\n",
    "# IMAGE_SIZE = 160\n",
    "# dataset_name = \"open-payments\"\n",
    "# target = \"status\"\n",
    "# IMAGE_SIZE = 96\n",
    "# dataset_name = \"bank-marketing\"\n",
    "# IMAGE_SIZE = 160\n",
    "# dataset_name = \"springleaf-marketing-response\"\n",
    "# IMAGE_SIZE = 1408\n",
    "# dataset_name = \"bnp-cardif\"\n",
    "# IMAGE_SIZE = 384\n",
    "# dataset_name = \"albert\"\n",
    "# IMAGE_SIZE = 288\n",
    "# dataset_name = \"titanic\"\n",
    "# IMAGE_SIZE = 128\n",
    "# target = \"Survived\"\n",
    "\n",
    "# IMAGE_SIZE = 24\n",
    "# dataset_name = \"rl\"\n",
    "# target = \"target\"\n",
    "# dataset_name = \"bank-marketing\"\n",
    "# target = \"y\"\n",
    "dataset_name = \"give-me-some-credit\"\n",
    "target = \"SeriousDlqin2yrs\"\n",
    "\n",
    "# dataset_name = \"census-income\"\n",
    "# target = \"taxable income amount\"\n",
    "\n",
    "# dataset_name = \"homesite-quote-conversion\"\n",
    "# target = \"QuoteConversion_Flag\"\n",
    "\n",
    "# dataset_name = \"springleaf-marketing-response\"\n",
    "# target = \"target\n",
    "# dataset_name = \"cat-in-the-dat-ii\"\n",
    "# IMAGE_SIZE = 96\n",
    "# dataset_name = \"give-me-some-credit\"\n",
    "# IMAGE_SIZE = 128\n",
    "\n",
    "# IMAGE_SIZE = 160\n",
    "\n",
    "DATASET_FOLDER = Path(os.getcwd()) / f\"data/{dataset_name}\"\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "PREFETCH = 50\n",
    "\n",
    "\n",
    "ONE_CHANNEL = True\n",
    "NB_CHANNEL = 1 if ONE_CHANNEL else 3\n",
    "\n",
    "# Target size for model to use\n",
    "# TARGET_SIZE = IMAGE_SIZE\n",
    "PRETRAINED_MODEL = EfficientNetB0  # MobileNetV2  # EfficientNetB0  # EfficientNetB4  # MobileNetV2  # EfficientNetB2  # EfficientNetB0  # MobileNetV2  # NASNetMobile\n",
    "\n",
    "# Training params for only new layers\n",
    "\n",
    "# Training params for refit\n",
    "epochs = 200\n",
    "patience = 5\n",
    "BATCH_SIZE = 32\n",
    "PREFETCH = 50\n",
    "\n",
    "# Model name => image size, last_block retrain\n",
    "# NASNetMobile => 224,\n",
    "# NASNetLarge => 331,\n",
    "# MobileNetV2 => 96, ... 160..... 224, last layer => 128 (3 blocks), 137(2 blocks), 146(1 block)\n",
    "# InceptionV3 => 299,\n",
    "# Xception => 299,"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-17T13:04:39.963383Z",
     "start_time": "2020-04-17T13:04:39.955914Z"
    }
   },
   "source": [
    "## Import + utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.276897Z",
     "start_time": "2020-09-15T07:10:05.167Z"
    }
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import imshow\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score\n",
    "\n",
    "from thc_net.image.pretrained_model import (\n",
    "    build_dataset,\n",
    "    build_process_path,\n",
    "    build_model,\n",
    ")\n",
    "\n",
    "from thc_net.image.tabular_preproc import csv_to_pixel\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.277478Z",
     "start_time": "2020-09-15T07:10:05.168Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_metric(history, metric):\n",
    "    # Plot training & validation loss values\n",
    "    plt.plot(history.history[metric])\n",
    "    plt.plot(history.history[f\"val_{metric}\"])\n",
    "    plt.title(f\"Model {metric}\")\n",
    "    plt.ylabel(f\"{metric}\")\n",
    "    plt.xlabel(\"Epoch\")\n",
    "    plt.legend([\"Train\", \"Test\"], loc=\"upper left\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.278074Z",
     "start_time": "2020-09-15T07:10:05.170Z"
    }
   },
   "outputs": [],
   "source": [
    "!ls /work/data/census-income"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-17T13:09:05.896414Z",
     "start_time": "2020-04-17T13:09:05.860854Z"
    }
   },
   "source": [
    "## Preparing datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.278604Z",
     "start_time": "2020-09-15T07:10:05.173Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "panda_kwargs = {}\n",
    "DATASET_FILENAME = \"train_bench.csv\"\n",
    "dataset_path = DATASET_FOLDER / DATASET_FILENAME\n",
    "TAILORED_COLUMN = \"Set\"\n",
    "\n",
    "columns = pd.read_csv(dataset_path, **panda_kwargs, nrows=1).columns.tolist()\n",
    "columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.279095Z",
     "start_time": "2020-09-15T07:10:05.174Z"
    }
   },
   "outputs": [],
   "source": [
    "split = None\n",
    "panda_kwargs = {}\n",
    "if TAILORED_COLUMN not in columns:\n",
    "    split = np.random.choice(\n",
    "        [\"train\", \"valid\", \"test\"], p=[0.8, 0.1, 0.1], size=(NB_LINES,)\n",
    "    )\n",
    "else:\n",
    "    split = pd.read_csv(\n",
    "        dataset_path, **panda_kwargs, usecols=[TAILORED_COLUMN]\n",
    "    ).values.reshape(-1)\n",
    "\n",
    "\n",
    "train_indices = np.argwhere(split == \"train\").reshape(-1)\n",
    "np.random.shuffle(train_indices)\n",
    "valid_indices = np.argwhere(split == \"valid\").reshape(-1)\n",
    "test_indices = np.argwhere(split == \"test\").reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.279558Z",
     "start_time": "2020-09-15T07:10:05.176Z"
    }
   },
   "outputs": [],
   "source": [
    "used_columns = list(set(columns) - set([TAILORED_COLUMN, target]))\n",
    "used_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.280008Z",
     "start_time": "2020-09-15T07:10:05.177Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(dataset_path, low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.280668Z",
     "start_time": "2020-09-15T07:10:05.178Z"
    }
   },
   "outputs": [],
   "source": [
    "X = df[used_columns].values\n",
    "Y = df[target].values.reshape(-1)\n",
    "\n",
    "extract_df = df[used_columns]\n",
    "nb_idx = (extract_df.dtypes == \"int64\") | (extract_df.dtypes == \"float64\")\n",
    "nb_idx = extract_df.columns[nb_idx]\n",
    "is_numeric = np.isin(extract_df.columns.values, nb_idx.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.281133Z",
     "start_time": "2020-09-15T07:10:05.181Z"
    }
   },
   "outputs": [],
   "source": [
    "(extract_df.nunique() / extract_df.shape[0]) < 0.02"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.281647Z",
     "start_time": "2020-09-15T07:10:05.182Z"
    }
   },
   "outputs": [],
   "source": [
    "# https://www.tensorflow.org/tutorials/load_data/images\n",
    "X, X_num, Y = csv_to_pixel(df, used_columns, target, ascii_only=ONE_CHANNEL)\n",
    "IMAGE_SIZE = X.shape[1]\n",
    "TARGET_SIZE = IMAGE_SIZE  # if IMAGE_SIZE > 32 else 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.282116Z",
     "start_time": "2020-09-15T07:10:05.184Z"
    }
   },
   "outputs": [],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.282631Z",
     "start_time": "2020-09-15T07:10:05.185Z"
    }
   },
   "outputs": [],
   "source": [
    "X_num.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.283097Z",
     "start_time": "2020-09-15T07:10:05.187Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.utils import to_categorical\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "Y = LabelEncoder().fit_transform(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.283541Z",
     "start_time": "2020-09-15T07:10:05.187Z"
    }
   },
   "outputs": [],
   "source": [
    "Y = to_categorical(Y.reshape(-1, 1))\n",
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.284002Z",
     "start_time": "2020-09-15T07:10:05.188Z"
    }
   },
   "outputs": [],
   "source": [
    "OUTPUT_DIM = len(np.unique(Y))\n",
    "OUTPUT_DIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.284471Z",
     "start_time": "2020-09-15T07:10:05.190Z"
    }
   },
   "outputs": [],
   "source": [
    "X.nbytes / 1024 ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.284994Z",
     "start_time": "2020-09-15T07:10:05.191Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.data import Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.285508Z",
     "start_time": "2020-09-15T07:10:05.191Z"
    }
   },
   "outputs": [],
   "source": [
    "epochs = 2000\n",
    "patience = 5\n",
    "BATCH_SIZE = 1024\n",
    "PREFETCH = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.285970Z",
     "start_time": "2020-09-15T07:10:05.193Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_train = (\n",
    "    Dataset.from_tensor_slices(\n",
    "        ((X[train_indices], X_num[train_indices]), Y[train_indices])\n",
    "    )\n",
    "    #     .repeat()\n",
    "    .batch(BATCH_SIZE)\n",
    ")\n",
    "dataset_valid = (\n",
    "    Dataset.from_tensor_slices(\n",
    "        ((X[valid_indices], X_num[valid_indices]), Y[valid_indices])\n",
    "    )\n",
    "    #     .repeat()\n",
    "    .batch(BATCH_SIZE)\n",
    ")\n",
    "dataset_test = Dataset.from_tensor_slices(\n",
    "    ((X[test_indices], X_num[test_indices]), Y[test_indices])\n",
    ").batch(BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.286434Z",
     "start_time": "2020-09-15T07:10:05.194Z"
    }
   },
   "outputs": [],
   "source": [
    "for (image, num), label in dataset_train.take(1):\n",
    "    print(\"Image shape: \", image.numpy().shape)\n",
    "    print(\"Num shape: \", num.numpy().shape)\n",
    "    print(\"Label: \", label.numpy().shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Images Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.286910Z",
     "start_time": "2020-09-15T07:10:05.195Z"
    }
   },
   "outputs": [],
   "source": [
    "def show_image(image, image_size, one_channel=False):\n",
    "    if one_channel:\n",
    "        imshow(image.reshape(IMAGE_SIZE, IMAGE_SIZE))\n",
    "    else:\n",
    "        imshow(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.287514Z",
     "start_time": "2020-09-15T07:10:05.195Z"
    }
   },
   "outputs": [],
   "source": [
    "for (image, num), label in dataset_train.take(1):\n",
    "    print(\"Label: \", label[0].numpy())\n",
    "    print(\"Label: \", label[0].numpy())\n",
    "    show_image(image[0].numpy(), IMAGE_SIZE, ONE_CHANNEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.288121Z",
     "start_time": "2020-09-15T07:10:05.197Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import (\n",
    "    Dense,\n",
    "    Conv2D,\n",
    "    Conv1D,\n",
    "    Flatten,\n",
    "    MaxPool2D,\n",
    "    Dropout,\n",
    "    BatchNormalization,\n",
    "    Input,\n",
    "    SpatialDropout2D,\n",
    "    Concatenate,\n",
    "    Reshape,\n",
    ")\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow_addons.optimizers import Lookahead, RectifiedAdam\n",
    "from tensorflow_addons.activations import mish, gelu\n",
    "from tensorflow.keras.activations import swish\n",
    "\n",
    "optimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)\n",
    "activation = \"relu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.288695Z",
     "start_time": "2020-09-15T07:10:05.198Z"
    }
   },
   "outputs": [],
   "source": [
    "inputs = Input(shape = (IMAGE_SIZE, IMAGE_SIZE, NB_CHANNEL))\n",
    "\n",
    "inputs_num = Input(shape = (X_num.shape[1]))\n",
    "x2 = Dense(X_num.shape[1]*5, activation = activation, kernel_initializer=\"he_normal\")(inputs_num)\n",
    "x2 = Dropout(rate=0.5)(x2)\n",
    "x2 = Dense(X_num.shape[1]*2, activation = activation, kernel_initializer=\"he_normal\")(x2)\n",
    "x2 = Dropout(rate=0.2)(x2)\n",
    "x2 = Dense(OUTPUT_DIM, activation=activation, kernel_initializer=\"he_normal\")(x2)\n",
    "\n",
    "#w2 = BatchNormalization()(inputs_num)\n",
    "\n",
    "x=inputs\n",
    "x = Conv2D(            \n",
    "    kernel_size= (1, 1),\n",
    "    padding= \"SAME\",\n",
    "    activation= \"relu\",\n",
    "    filters=32\n",
    ")(inputs)\n",
    "# x = SpatialDropout2D(0.5)(x)\n",
    "\n",
    "# x = BatchNormalization()(x)\n",
    "\n",
    "# x = Conv2D(            \n",
    "#     kernel_size= (8, 8),\n",
    "#     strides=(8,8),\n",
    "#     padding= \"SAME\",\n",
    "#     activation=activation,\n",
    "#     kernel_initializer=\"he_normal\",\n",
    "#     filters=32\n",
    "# )(x)\n",
    "# x = Conv2D(            \n",
    "#     kernel_size= (4, 4),\n",
    "#     strides=(4, 4),\n",
    "#     padding= \"SAME\",\n",
    "#     activation=activation,\n",
    "#     kernel_initializer=\"he_normal\",\n",
    "#     filters=64\n",
    "# )(x)\n",
    "\n",
    "# x = Conv2D(            \n",
    "#     kernel_size= (8,8),\n",
    "#     strides=(1, 1),\n",
    "#     padding= \"SAME\",\n",
    "#     activation=activation,\n",
    "#     kernel_initializer=\"he_normal\",\n",
    "#     filters=128\n",
    "# )(x)\n",
    "# x = SpatialDropout2D(0.5)(x)\n",
    "\n",
    "x = Conv2D(            \n",
    "    kernel_size= (8,8),\n",
    "    strides=(8,8),\n",
    "    padding= \"SAME\",\n",
    "    activation=activation,\n",
    "#     kernel_initializer=\"he_normal\",\n",
    "    filters=16\n",
    ")(x)\n",
    "x = SpatialDropout2D(0.2)(x)\n",
    "\n",
    "# x = Conv2D(            \n",
    "#     kernel_size= (IMAGE_SIZE//8, IMAGE_SIZE//8),\n",
    "#     strides=(1,1),\n",
    "#     padding= \"SAME\",\n",
    "#     activation=activation,\n",
    "# #     kernel_initializer=\"he_normal\",\n",
    "#     filters=32\n",
    "# )(x)\n",
    "\n",
    "# x = Reshape(((IMAGE_SIZE//8)*(IMAGE_SIZE//8), 16))(x)\n",
    "# x = Conv1D(            \n",
    "#     kernel_size=16,\n",
    "#     strides=1,\n",
    "#     padding= \"SAME\",\n",
    "#     activation=activation,\n",
    "# #     kernel_initializer=\"he_normal\",\n",
    "#     filters=16\n",
    "# )(x)\n",
    "\n",
    "# x = MaxPool2D(pool_size=(4,4))(x)\n",
    "# x = SpatialDropout2D(0.3)(x)\n",
    "\n",
    "\n",
    "# x = Conv2D(            \n",
    "#     kernel_size= (4, 4),\n",
    "#     strides=(1, 1),\n",
    "#     padding= \"SAME\",\n",
    "#     activation= \"relu\",\n",
    "#     filters=128\n",
    "# )(x)\n",
    "\n",
    "# x = Conv2D(            \n",
    "#     kernel_size= (4, 4),\n",
    "#     strides=(4, 4),\n",
    "#     padding= \"SAME\",\n",
    "#     activation= \"relu\",\n",
    "#     filters=256\n",
    "# )(x)\n",
    "\n",
    "# x = BatchNormalization()(x)\n",
    "# x = Conv2D(            \n",
    "#     kernel_size= (1, 1),\n",
    "#     padding= \"SAME\",\n",
    "#     activation= \"relu\",\n",
    "#     filters=128\n",
    "# )(x)\n",
    "\n",
    "\n",
    "# x = Conv2D(            \n",
    "#     kernel_size= (1, 1),\n",
    "#     padding= \"SAME\",\n",
    "#     activation= \"relu\",\n",
    "#     filters=32\n",
    "# )(x)\n",
    "\n",
    "# x = Conv2D(            \n",
    "#     kernel_size= (1, 1),\n",
    "#     padding= \"SAME\",\n",
    "#     activation= \"relu\",\n",
    "#     filters=16\n",
    "# )(x)\n",
    "\n",
    "# x = MaxPool2D(pool_size=(4,4))(x)\n",
    "x = Flatten()(x)\n",
    "x = Dense(OUTPUT_DIM, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "# x = BatchNormalization()(x)\n",
    "\n",
    "# x = BatchNormalization()(x)\n",
    "# x2 = BatchNormalization()(x2)\n",
    "x = Concatenate()([x, x2])\n",
    "\n",
    "x = Dense(16, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "# x = Dense((X_num.shape[1]*5+IMAGE_SIZE//8*8)*2, activation = activation, kernel_initializer=\"he_normal\")(x)\n",
    "\n",
    "# # x = Dropout(rate=0.2)(x)\n",
    "# x = Dense((X_num.shape[1]*5+IMAGE_SIZE//8*8), activation = activation, kernel_initializer=\"he_normal\")(x)\n",
    "# # x = Dropout(rate=0.2)(x)\n",
    "\n",
    "# if (X_num.shape[1]*5+IMAGE_SIZE//8*8) > 512:\n",
    "#     x = Dense(128, activation = activation, kernel_initializer=\"he_normal\")(x)\n",
    "#     x = Dropout(rate=0.2)(x)\n",
    "\n",
    "# x = Dense(512, activation = \"relu\")(x)\n",
    "# x = Dropout(rate=0.2)(x)\n",
    "\n",
    "# x = Dense(128, activation = \"relu\")(x)\n",
    "# x = Dropout(rate=0.2)(x)\n",
    "\n",
    "x = Dense(OUTPUT_DIM, activation = 'sigmoid')(x)\n",
    "\n",
    "model = Model(inputs=[inputs, inputs_num], outputs=x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.289208Z",
     "start_time": "2020-09-15T07:10:05.198Z"
    }
   },
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.289732Z",
     "start_time": "2020-09-15T07:10:05.199Z"
    }
   },
   "outputs": [],
   "source": [
    "#!pip install pydot graphviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.290178Z",
     "start_time": "2020-09-15T07:10:05.200Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.utils import model_to_dot, plot_model\n",
    "from IPython.display import SVG"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-12T07:41:44.067736Z",
     "start_time": "2020-05-12T07:41:43.808349Z"
    }
   },
   "source": [
    "SVG(\n",
    "    model_to_dot(\n",
    "        model,\n",
    "        show_shapes=True,\n",
    "        show_layer_names=True,\n",
    "        rankdir=\"TB\",\n",
    "        expand_nested=False,\n",
    "        dpi=96,\n",
    "        subgraph=False,\n",
    "    ).create(prog=\"dot\", format=\"svg\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.290686Z",
     "start_time": "2020-09-15T07:10:05.223Z"
    }
   },
   "outputs": [],
   "source": [
    "# from keras.utils import plot_model\n",
    "plot_model(\n",
    "    model, show_shapes=True, show_layer_names=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model, using existing one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.291214Z",
     "start_time": "2020-09-15T07:10:05.224Z"
    }
   },
   "outputs": [],
   "source": [
    "es = EarlyStopping(\n",
    "    monitor=\"val_loss\",\n",
    "    verbose=1,\n",
    "    mode=\"min\",\n",
    "    patience=patience,\n",
    "    restore_best_weights=True,\n",
    ")\n",
    "# compile the model (should be done *after* setting layers to non-trainable)\n",
    "\n",
    "# We need to recompile the model for these modifications to take effect\n",
    "es.set_model(model)\n",
    "loss = \"binary_crossentropy\" if OUTPUT_DIM == 2 else \"categorical_crossentropy\"\n",
    "# optimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)\n",
    "\n",
    "model.compile(optimizer=optimizer, loss=loss)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-05-11T11:20:49.357Z"
    }
   },
   "source": [
    "model, callbacks = build_model(\n",
    "    EfficientNetB0,\n",
    "    image_size=IMAGE_SIZE,\n",
    "    nb_channel=NB_CHANNEL,\n",
    "    input_size=TARGET_SIZE,  # IMAGE_SIZE,\n",
    "    output_dim=OUTPUT_DIM,\n",
    "    weights=None,\n",
    "    patience=5,\n",
    ")"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-05-11T11:20:49.361Z"
    }
   },
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fit new layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.291734Z",
     "start_time": "2020-09-15T07:10:05.257Z"
    }
   },
   "outputs": [],
   "source": [
    "truth_test = []\n",
    "for (_, _), labels in dataset_test:\n",
    "    truth_test.append(np.argmax(labels, axis=1))\n",
    "truth_test = np.hstack(truth_test)\n",
    "truth_test\n",
    "\n",
    "truth_valid = []\n",
    "for i, ((_, _), labels) in enumerate(dataset_valid):\n",
    "    truth_valid.append(np.argmax(labels, axis=1))\n",
    "#     if i >= steps_per_epoch_val - 1:\n",
    "#         break\n",
    "truth_valid = np.hstack(truth_valid)\n",
    "truth_valid.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unfreeze and fit more/all layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.292221Z",
     "start_time": "2020-09-15T07:10:05.259Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# we train our model again (this time fine-tuning the top 2 inception blocks\n",
    "# alongside the top Dense layers\n",
    "history = model.fit(\n",
    "    dataset_train,\n",
    "    callbacks=[es],\n",
    "    epochs=epochs,\n",
    "    # steps_per_epoch=steps_per_epoch,\n",
    "    validation_data=dataset_valid,\n",
    "    # validation_steps=steps_per_epoch_val,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.292716Z",
     "start_time": "2020-09-15T07:10:05.260Z"
    }
   },
   "outputs": [],
   "source": [
    "plot_metric(history, \"loss\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-17T13:16:20.349217Z",
     "start_time": "2020-04-17T13:16:16.612Z"
    }
   },
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.293348Z",
     "start_time": "2020-09-15T07:10:05.261Z"
    }
   },
   "outputs": [],
   "source": [
    "preds_valid = model.predict(dataset_valid)  # , steps=steps_per_epoch_val)\n",
    "preds_valid.shape\n",
    "if OUTPUT_DIM > 2:\n",
    "    print(\n",
    "        f\"Accuracy valid: {accuracy_score(truth_valid, np.argmax(preds_valid, axis=1))}\"\n",
    "    )\n",
    "if OUTPUT_DIM == 2:\n",
    "    print(f\"ROC AUC valid: {roc_auc_score(truth_valid, preds_valid[:, 1])}\")\n",
    "preds_test = model.predict(dataset_test)\n",
    "preds_test.shape\n",
    "if OUTPUT_DIM > 2:\n",
    "    print(f\"Accuracy test: {accuracy_score(truth_test, np.argmax(preds_test, axis=1))}\")\n",
    "if OUTPUT_DIM == 2:\n",
    "    print(f\"ROC AUC test: {roc_auc_score(truth_test, preds_test[:, 1])}\")"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Bank martketing No numeric\n",
    "ROC AUC valid: 0.7849241489978159\n",
    "ROC AUC test: 0.8066209120243591\n",
    "\n",
    "With numeric\n",
    "ROC AUC valid: 0.7415022647293126\n",
    "ROC AUC test: 0.7471501527935644"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.293906Z",
     "start_time": "2020-09-15T07:10:05.275Z"
    }
   },
   "outputs": [],
   "source": [
    "# Open payments 13min, 0.9338321957865205 (batch 256)\n",
    "# cat-in-the-dat-ii 0.7643929251726093 1h10 (batch 256)\n",
    "# RL ROC AUC test: 0.9430279072306678 mobile net patience 20\n",
    "# BNP cardif (image size : 96) ROC AUC test: 0.7168011573597879 1h16\n",
    "# give-me-some-credit ROC AUC test: 0.8591709381962962 2h4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.397650Z",
     "start_time": "2020-09-15T07:10:05.394917Z"
    }
   },
   "outputs": [],
   "source": [
    "# Open payments 1h10\n",
    "# ROC AUC valid: 0.9432323040315754\n",
    "# ROC AUC test: 0.9345809216081824"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.401574Z",
     "start_time": "2020-09-15T07:10:05.398474Z"
    }
   },
   "outputs": [],
   "source": [
    "# EfficientNetB0\n",
    "# EfficientNetB2 2h => ROC AUC valid: 0.9357130861335854 ROC AUC test: 0.9330937133279599"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.406288Z",
     "start_time": "2020-09-15T07:10:05.402491Z"
    }
   },
   "outputs": [],
   "source": [
    "# \"santander-customer-satisfaction\"\n",
    "# 0.8164865598696714 => target size 96, whole re train\n",
    "# 0.8141824599511267 => target size 224, whole re train\n",
    "# ROC AUC valid: 0.8268400760249797 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# give me some credit\n",
    "# 0.8462134942186483 => target size 160, whole train (batch 128, 2layers 1024, 128)\n",
    "\n",
    "# ROC AUC valid: 0.8348917439829162\n",
    "# RL\n",
    "# ROC AUC valid: 0.892118469133795 => 160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.9233534348199217 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9465346534653467 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# Open payment\n",
    "# ROC AUC valid: 0.9186458210299415 => 96 =>160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.8778868370932499 => 96 => 96, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.8917381493730192 => 96 => 224, whole, batch 64, 1layer 1024\n",
    "# ROC AUC valid: 0.9090374872044725 => 96 => 96, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC valid: 0.886417393797122 => 96 => 160, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC valid: 0.9045819676568436 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9045819676568436 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9406275221953189 => 96 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "# ROC AUC valid: 0.9399110034154216 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "# ROC AUC valid: 0.8811034128677376 => 96 => 96, whole, batch 128, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# Bank marketing\n",
    "# ROC AUC valid: 0.7970734141661526 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# Albert\n",
    "# ROC AUC valid: 0.7500980687987842 => 288 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# bnp-cardif\n",
    "# ROC AUC valid: 0.7206667869818926 => 384 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T07:10:05.411737Z",
     "start_time": "2020-09-15T07:10:05.407119Z"
    }
   },
   "outputs": [],
   "source": [
    "# \"santander-customer-satisfaction\"\n",
    "# 0.833417731838137 => target size 96, whole train\n",
    "# 0.8170226029745679 => target size 224, whole train\n",
    "# ROC AUC test: 0.8348549041045967 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# give me some credit\n",
    "# 0.8453497574694486 => target size 160, whole train (batch 128, 2layers 1024, 128)\n",
    "# ROC AUC test: 0.8447058873195916\n",
    "\n",
    "# RL\n",
    "# ROC AUC test: 0.9051288159651395 => 160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.9128674518211912 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "# ROC AUC test: 0.9456874816987527 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "# ROC AUC test: 0.9399110034154216 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout, earlystoping eevry time\n",
    "\n",
    "# Open payment\n",
    "# ROC AUC test: 0.916907759155486 => 96 =>160, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.8931101859362467 => 96 => 96, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.8938387451368033 => 96 => 224, whole, batch 64, 1layer 1024\n",
    "# ROC AUC test: 0.9067445823812874 => 96 => 96, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC test: 0.8954549557710788 => 96 => 160, whole, batch 64, 1layer 1024 -> 512 -> 128\n",
    "# ROC AUC test: 0.9062895529860363 => 96 => 96, whole, batch 64, 1layer 1024 -> 128, no dropout\n",
    "# ROC AUC test: 0.9403517762951931 => 96 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# Bank marketing\n",
    "# ROC AUC valid: 0.7959000291791145 => 160 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# Albert\n",
    "# ROC AUC test: 0.7487660412524685 => 288 => 160, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n",
    "\n",
    "# bnp-cardif\n",
    "# ROC AUC test: 0.725934546476426 => 384 => 96, whole, batch 32, 1layer 1024 -> 512 -> 128, no dropout\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "notify_time": "5"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}