FCN_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\" \n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"-1\"\n",
    "# Uncomment the above 2 lines if you want to use your CPU instead of GPU\n",
    "\n",
    "import time\n",
    "import numpy as np\n",
    "import sys\n",
    "import gc\n",
    "\n",
    "import keras\n",
    "print(\"Keras version:\", keras.__version__)\n",
    "\n",
    "import tensorflow as tf\n",
    "print(\"TensorFlow version:\", tf.__version__)\n",
    "\n",
    "from PIL import ImageFile\n",
    "# the following will allow \"damaged\" image files to be loaded without an exception firing\n",
    "ImageFile.LOAD_TRUNCATED_IMAGES = True \n",
    "\n",
    "from keras.models import Model\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "from keras.layers.convolutional import Convolution2D\n",
    "from keras.layers.pooling import MaxPooling2D\n",
    "from keras.layers.pooling import GlobalMaxPooling2D\n",
    "from keras.layers.pooling import GlobalAveragePooling2D\n",
    "from keras.engine import InputLayer\n",
    "\n",
    "from keras.preprocessing.image import load_img\n",
    "from keras.preprocessing.image import img_to_array\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "# The following cell will test whether the extended version of \n",
    "# keras' \"load_img\" method was successfully installed and is callable.\n",
    "print('\\nload_img tests::')\n",
    "from keras.preprocessing.image import set_load_img_type\n",
    "set_load_img_type(0) # prints: 'load_img' calls 'load_img_keras'\n",
    "set_load_img_type(1) # prints: 'load_img' calls 'load_img_pad'\n",
    "set_load_img_type(2) # prints: 'load_img' calls 'load_img_pad' (+mask)\n",
    "set_load_img_type(3) # prints: 'load_img' calls 'load_img_crop'\n",
    "set_load_img_type(4) # prints: 'load_img' calls 'load_img_multicrop'\n",
    "im = load_img('test_image.jpg')\n",
    "ar = im.size[1]/im.size[0]\n",
    "print('    using none        -> ', im.size, '(aspect ratio = %.3f)' % ar)\n",
    "im = load_img('test_image.jpg', target_size = (350,290))\n",
    "ar = im.size[1]/im.size[0]\n",
    "print('    using (290,350)   -> ', im.size, '(aspect ratio = %.3f)' % ar)\n",
    "im = load_img('test_image.jpg', target_size = (None,500), interpolation='nearest')\n",
    "ar = im.size[1]/im.size[0]\n",
    "print('    using (500,None)  -> ', im.size, '(aspect ratio = %.3f)' % ar)\n",
    "im = load_img('test_image.jpg', target_size = (500,None))\n",
    "ar = im.size[1]/im.size[0]\n",
    "print('    using (None,500)  -> ', im.size, '(aspect ratio = %.3f)' % ar)\n",
    "im = load_img('test_image.jpg', target_size = (None,100))\n",
    "ar = im.size[1]/im.size[0]\n",
    "print('    using (100,None)  -> ', im.size, '(aspect ratio = %.3f)' % ar)\n",
    "im = load_img('test_image.jpg', target_size = (100,None))\n",
    "ar = im.size[1]/im.size[0]\n",
    "print('    using (None,100)  -> ', im.size, '(aspect ratio = %.3f)' % ar)\n",
    "im = load_img('test_image.jpg', target_size = (None,None))\n",
    "ar = im.size[1]/im.size[0]\n",
    "print('    using (None,None) -> ', im.size, '(aspect ratio = %.3f)' % ar)\n",
    "set_load_img_type(0) # reset to default 'load_img' method\n",
    "\n",
    "# model architectures\n",
    "from keras.applications import vgg16\n",
    "from keras.applications import vgg19\n",
    "from keras.applications import inception_v3\n",
    "from keras.applications import xception\n",
    "\n",
    "# model FCN architectures\n",
    "from keras.applications import vgg16_fcn\n",
    "from keras.applications import vgg19_fcn\n",
    "from keras.applications import inception_v3_fcn\n",
    "from keras.applications import xception_fcn\n",
    "\n",
    "# Seed the Random Number Generators\n",
    "from numpy.random import seed\n",
    "seed(1)\n",
    "from tensorflow import set_random_seed\n",
    "set_random_seed(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup parameters\n",
    "Everything you need to change for your own experiments (parameters, location of train and evaluation datasets) is in the following cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_DATA_DIR = 'kagglecatsanddogs_3367a/PetImagesTrain'\n",
    "VALIDATION_DATA_DIR = 'kagglecatsanddogs_3367a/PetImagesTest'\n",
    "DATASET_NAME = 'PetImages'\n",
    "\n",
    "TRAIN_DATA_DIR = 'AE_AVA2_TRAIN'\n",
    "VALIDATION_DATA_DIR = 'AE_AVA2_TEST'\n",
    "DATASET_NAME = 'AVA2'\n",
    "\n",
    "OUTPUT_DIR = 'snapshots'\n",
    "NO_OF_CLASSES = 2\n",
    "RATIO_FOR_SMALL_VALIDATION = 0.01\n",
    "BATCH_SIZE = 8\n",
    "LR_START = 0.01\n",
    "LR_MIN = 0.001\n",
    "FREEZE_UP_TO = 0\n",
    "DROPOUT_RATE = 0.0\n",
    "\n",
    "EPOCHS_OF_FULL_EVAL = [5, 10, 15, 20, 30, 40, 45, 50, 55, 60]\n",
    "EPOCHS_NO = 20\n",
    "\n",
    "MODEL_NAME = 'v16'  # use 'v16' for VGG16, 'v19' for VGG19\n",
    "                    # 'x' for Xception and 'i' for Inception V3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare session\n",
    "if MODEL_NAME == \"v16\":\n",
    "    target_size_ = 224\n",
    "if MODEL_NAME == \"v19\":\n",
    "    target_size_ = 224 \n",
    "if MODEL_NAME == \"i\":\n",
    "    target_size_ = 299\n",
    "if MODEL_NAME == \"x\":\n",
    "    target_size_ = 299\n",
    "\n",
    "if not os.path.exists(OUTPUT_DIR):\n",
    "    os.makedirs(OUTPUT_DIR)\n",
    "\n",
    "info_string = MODEL_NAME + \\\n",
    "            \"-d\" + DATASET_NAME + \\\n",
    "            \"-b\" + str(BATCH_SIZE) + \\\n",
    "            \"-d\" + str(DROPOUT_RATE)\n",
    "\n",
    "print('Session info :', info_string)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Definition of main functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_exists(str_name):\n",
    "    for i in os.listdir(OUTPUT_DIR):\n",
    "        if os.path.isfile(os.path.join(OUTPUT_DIR,i)) and i.startswith(str_name):\n",
    "            return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, run_id, train_flow, val_flow, force_overwrite=False):\n",
    "    # Setup a common training  procedure. \n",
    "    #\n",
    "    # For this, we consider two validation sets:\n",
    "    # 1. A small validation set that uses a RATIO_FOR_SMALL_VALIDATION ratio of the test set\n",
    "    # 2. A large validation set that uses the whole test set, \n",
    "    #    so that the accuracy reported is the test accuracy\n",
    "    #\n",
    "    # We then train the model in the following stages:\n",
    "    # 1. 29 epochs (epochs 1 to 29) using the small validation set\n",
    "    # 2. 1 epoch (epoch 30) using the large validation set\n",
    "    # 3. 9 epochs (epochs 31 to 39) using the small validation set\n",
    "    # 4. 1 epoch (epoch 40) using the large validation set\n",
    "    # 5. 4 epochs (epochs 41 to 44) using the small validation set\n",
    "    # 6. 1 epoch (epoch 45) using the large validation set\n",
    "    # 7. 4 epochs (epochs 41 to 44) using the small validation set\n",
    "    # 8. 1 epoch (epoch 50) using the large validation set\n",
    "    # 9. 4 epochs (epochs 51 to 54) using the small validation set\n",
    "    # 10. 1 epoch (epoch 55) using the large validation set\n",
    "    # 11. 4 epochs (epochs 56 to 59) using the small validation set\n",
    "    # 12. 1 epoch (epoch 60) using the large validation set\n",
    "    #\n",
    "    # Each stage has its own logger so we can see the course of the whole training procedure,\n",
    "    # and epochs 30, 40, 45, 50, 55 and 60 report the test accuracy.\n",
    "    #\n",
    "    # The reuslting model weights are saved only for epochs where the full validation set is used\n",
    "    # (i.e. epochs 30, 40, 45, 50, 55 and 60)\n",
    "    # \n",
    "    \n",
    "    # use the global info string\n",
    "    global info_string\n",
    "\n",
    "    if (model_exists(info_string + '_' + run_id + '_')) and not force_overwrite:\n",
    "        print(\" Output for '%s' exists.\" % (info_string + '_' + run_id + '_'))\n",
    "        print(\" Training skipped.\")\n",
    "        return \n",
    "    \n",
    "    print(\"    Session %s\" % (info_string + '_' + run_id + '_'))\n",
    "\n",
    "    # Freeze the layers which we don't want to be trained\n",
    "    for layer in model.layers[:FREEZE_UP_TO]:\n",
    "        layer.trainable = False\n",
    "        \n",
    "    # setup optimizer configuration\n",
    "    sgd = optimizers.SGD(lr=LR_START, momentum = 0.0)\n",
    "    \n",
    "    # compile the model with a SGD/momentum optimizer and a starting learning rate.\n",
    "    model.compile(loss='categorical_crossentropy', \n",
    "                  optimizer=sgd, \n",
    "                  metrics=['accuracy'])\n",
    "    \n",
    "    # setup model save callback function\n",
    "    cb_save_model = keras.callbacks.ModelCheckpoint(os.path.join(OUTPUT_DIR,\n",
    "                    info_string + '_' + run_id + '_' + 'e{epoch:02d}_acc{val_acc:.4f}.hdf5'), \n",
    "                    monitor='val_acc', verbose=1, \n",
    "                    save_best_only=True, save_weights_only=False, \n",
    "                    mode='auto', period=1)\n",
    "    \n",
    "    # setup \"reduce learning rate\" callback function\n",
    "    cb_reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='acc', factor=0.2, verbose=1,\n",
    "                                                     patience=3, min_delta=1e-3, \n",
    "                                                     cooldown=0, min_lr=LR_MIN)\n",
    "    \n",
    "    # set infer steps per epoch\n",
    "    spe = len(train_flow)\n",
    "    \n",
    "    # start training\n",
    "    print('  ')\n",
    "    for i in range(1,EPOCHS_NO+1):\n",
    "        if i in EPOCHS_OF_FULL_EVAL:\n",
    "            print(' (Epoch %02d - Fine-tuning using all of the evaluation data)' % i)\n",
    "            print(' -------------------------------------------------------------')\n",
    "            cb_csv_logger = keras.callbacks.CSVLogger(\n",
    "                                os.path.join(OUTPUT_DIR,info_string+'_'+run_id+'_training_'+str(i)+'.log'))\n",
    "            model.fit_generator(train_flow, steps_per_epoch=spe, shuffle=True, \n",
    "                                epochs=1, \n",
    "                                validation_data=val_flow, \n",
    "                                validation_steps=int(val_flow.samples),\n",
    "                                callbacks = [cb_reduce_lr, cb_save_model, cb_csv_logger])\n",
    "        else:\n",
    "            print(' (Epoch %02d - Fine-tuning using a small set of validation data)' % i)\n",
    "            print(' -----------------------------------------------------------------')\n",
    "            cb_csv_logger = keras.callbacks.CSVLogger(\n",
    "                                os.path.join(OUTPUT_DIR,info_string+'_'+run_id+'_training_'+str(i)+'.log'))\n",
    "            model.fit_generator(train_flow, steps_per_epoch=spe, shuffle=True, \n",
    "                                epochs=1, \n",
    "                                validation_data=val_flow, \n",
    "                                validation_steps=int(RATIO_FOR_SMALL_VALIDATION*val_flow.samples),\n",
    "                                callbacks = [cb_reduce_lr, cb_save_model, cb_csv_logger])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def setup_model(fcn=False):\n",
    "    global target_size_\n",
    "    if MODEL_NAME == \"v16\":\n",
    "        if not fcn:\n",
    "            print(\"Model VGG16\")\n",
    "            model = vgg16.VGG16(include_top=False, weights='imagenet', input_shape=[target_size_,target_size_,3])\n",
    "            x = model.output\n",
    "            print(\"    adding flatten layer...\") \n",
    "            x = layers.Flatten(name='flatten')(x)\n",
    "            print(\"    adding FC layer...\") \n",
    "            x = layers.Dense(4096, activation='relu', name='fc1')(x)\n",
    "            print(\"    adding FC layer...\") \n",
    "            x = layers.Dense(4096, activation='relu', name='fc2')(x)\n",
    "            if DROPOUT_RATE>0.01:\n",
    "                print(\"    adding Dropout layer (rate=.%3f)...\" % DROPOUT_RATE) \n",
    "                x = layers.Dropout(DROPOUT_RATE)(x)\n",
    "            print(\"    adding FC layer for %d classes...\" % NO_OF_CLASSES) \n",
    "            x = layers.Dense(NO_OF_CLASSES, activation='softmax', name='predictions')(x)\n",
    "            model = Model(model.input, x, name='vgg16')\n",
    "            print(\"    compiled 'VGG16 model.\") \n",
    "        else:\n",
    "            print(\"Model VGG16 FCN\")\n",
    "            model = vgg16_fcn.VGG16_fcn(weights='imagenet', input_shape=[None,None,3], classes=NO_OF_CLASSES)\n",
    "            x = model.output\n",
    "            if DROPOUT_RATE>0.01:\n",
    "                print(\"    adding Dropout layer (rate=.%3f)...\" % DROPOUT_RATE) \n",
    "                x = keras.layers.Dropout(DROPOUT_RATE)(x)\n",
    "            print(\"    adding GlobalMaxPooling2d layer...\")\n",
    "            x = GlobalMaxPooling2D()(x)\n",
    "            model = Model(model.input, x, name='VGG16_FCN')\n",
    "            print(\"    compiled 'VGG16_FCN' model.\")\n",
    "    if MODEL_NAME == \"v19\":\n",
    "        if not fcn:\n",
    "            print(\"Model VGG19\")\n",
    "            model = vgg19.VGG19(include_top=False, weights='imagenet',input_shape=[target_size_,target_size_,3])\n",
    "            x = model.output\n",
    "            print(\"    adding flatten layer...\") \n",
    "            x = layers.Flatten(name='flatten')(x)\n",
    "            print(\"    adding FC layer...\") \n",
    "            x = layers.Dense(4096, activation='relu', name='fc1')(x)\n",
    "            print(\"    adding FC layer...\") \n",
    "            x = layers.Dense(4096, activation='relu', name='fc2')(x)\n",
    "            if DROPOUT_RATE>0.01:\n",
    "                print(\"    adding Dropout layer (rate=.%3f)...\" % DROPOUT_RATE) \n",
    "                x = layers.Dropout(DROPOUT_RATE)(x)\n",
    "            print(\"    adding FC layer for %d classes...\" % NO_OF_CLASSES) \n",
    "            x = layers.Dense(NO_OF_CLASSES, activation='softmax', name='predictions')(x)\n",
    "            model = Model(model.input, x, name='vgg19')\n",
    "            print(\"    compiled 'VGG19 model.\") \n",
    "        else:\n",
    "            print(\"Model VGG19 FCN\")\n",
    "            model = vgg19_fcn.VGG19_fcn(weights='imagenet', input_shape=[None,None,3], classes=NO_OF_CLASSES)\n",
    "            x = model.output\n",
    "            if DROPOUT_RATE>0.01:\n",
    "                print(\"    adding Dropout layer (rate=.%3f)...\" % DROPOUT_RATE) \n",
    "                x = keras.layers.Dropout(DROPOUT_RATE)(x)\n",
    "            print(\"    adding GlobalMaxPooling2d layer...\")\n",
    "            x = GlobalMaxPooling2D()(x)\n",
    "            model = Model(model.input, x, name='VGG19_FCN')\n",
    "            print(\"    compiled 'VGG19_FCN' model.\")\n",
    "\n",
    "    if MODEL_NAME == \"i\":\n",
    "        if not fcn:\n",
    "            print(\"Model InceptionV3\")\n",
    "            model = inception_v3.InceptionV3(include_top=False, weights='imagenet', classes=NO_OF_CLASSES)\n",
    "            x = model.output\n",
    "            print(\"    adding GlobalMaxPooling2d layer...\")\n",
    "            x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n",
    "            if DROPOUT_RATE>0.01:\n",
    "                print(\"    adding Dropout layer (rate=.%3f)...\" % DROPOUT_RATE) \n",
    "                x = layers.Dropout(DROPOUT_RATE)(x)\n",
    "            print(\"    adding FC layer for %d classes...\" % NO_OF_CLASSES) \n",
    "            x = layers.Dense(classes, activation='softmax', name='predictions')(x)\n",
    "            model = Model(model.input, x, name='InceptionV3')\n",
    "            print(\"    compiled 'InceptionV3 model.\") \n",
    "        else:\n",
    "            print(\"Model InceptionV3 FCN\")\n",
    "            model = inception_v3_fcn.InceptionV3_fcn(weights='imagenet', classes=NO_OF_CLASSES)\n",
    "            x = model.output\n",
    "            if DROPOUT_RATE>0.01:\n",
    "                print(\"    adding Dropout layer (rate=.%3f)...\" % DROPOUT_RATE) \n",
    "                x = keras.layers.Dropout(DROPOUT_RATE)(x)\n",
    "            print(\"    adding GlobalMaxPooling2d layer...\")\n",
    "            x = GlobalMaxPooling2D()(x)\n",
    "            model = Model(model.input, x, name='InceptionV3_FCN')\n",
    "            print(\"    compiled 'InceptionV3_FCN' model.\")\n",
    "    if MODEL_NAME == \"x\":\n",
    "        if not fcn:\n",
    "            print(\"Model Xception\")\n",
    "            model = xception.Xception(include_top=False, weights='imagenet', classes=NO_OF_CLASSES)\n",
    "            x = model.output\n",
    "            print(\"    adding GlobalMaxPooling2d layer...\")\n",
    "            x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n",
    "            if DROPOUT_RATE>0.01:\n",
    "                print(\"    adding Dropout layer (rate=.%3f)...\" % DROPOUT_RATE) \n",
    "                x = layers.Dropout(DROPOUT_RATE)(x)\n",
    "            print(\"    adding FC layer for %d classes...\" % NO_OF_CLASSES) \n",
    "            x = layers.Dense(NO_OF_CLASSES, activation='softmax', name='predictions')(x)\n",
    "            model = Model(model.input, x, name='Xception')\n",
    "            print(\"    compiled 'Xception model.\") \n",
    "        else:\n",
    "            print(\"Model Xception FCN\")\n",
    "            model = xception_fcn.Xception_fcn(weights='imagenet', classes=NO_OF_CLASSES)\n",
    "            model = inception_v3_fcn.InceptionV3_fcn(weights='imagenet', classes=NO_OF_CLASSES)\n",
    "            x = model.output\n",
    "            if DROPOUT_RATE>0.01:\n",
    "                print(\"    adding Dropout layer (rate=.%3f)...\" % DROPOUT_RATE) \n",
    "                x = keras.layers.Dropout(DROPOUT_RATE)(x)\n",
    "            print(\"    adding GlobalMaxPooling2d layer...\")\n",
    "            x = GlobalMaxPooling2D()(x)\n",
    "            model = Model(model.input, x, name='Xception_FCN')\n",
    "            print(\"    compiled 'Xception_FCN' model.\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def setup_data_flows(upscale_ratio=1.0):\n",
    "    global target_size_\n",
    "    \n",
    "    # prepare data generators\n",
    "    print(\"Setting up train data generators...\")\n",
    "    train_data = ImageDataGenerator(rescale=1./255, horizontal_flip=True)\n",
    "    train_flow = train_data.flow_from_directory(\n",
    "                 TRAIN_DATA_DIR, target_size=(int(upscale_ratio*target_size_), int(upscale_ratio*target_size_)),\n",
    "                 batch_size=BATCH_SIZE, class_mode='categorical')\n",
    "\n",
    "    print(\"Setting up test data generators...\")\n",
    "    test_data = ImageDataGenerator(rescale=1./255)\n",
    "    test_flow = test_data.flow_from_directory(\n",
    "                VALIDATION_DATA_DIR, target_size=(int(upscale_ratio*target_size_), int(upscale_ratio*target_size_)), \n",
    "                batch_size=BATCH_SIZE, class_mode='categorical')\n",
    "\n",
    "    return train_flow, test_flow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiments\n",
    "\n",
    "In this section the following experiments are performed:\n",
    "* Experiment #1 - Finetune the original model.\n",
    "* Experiment #2 - Create an FCN version and finetune using the original input size.  \n",
    "* Experiment #3 - Finetune the FCN model using 1.5x of the original input size.  \n",
    "* Experiment #4 - Finetune the FCN model using 2.0x of the original input size.  \n",
    "* Experiment #5 - Finetune the FCN model using 3.0x of the original input size.  \n",
    "* Experiment #6 - Finetune the FCN model using 1.5x of the original input size (cropping method). \n",
    "* Experiment #7 - Finetune the FCN model using 1.5x of the original input size (padding method). \n",
    "* Experiment #8 - Finetune the FCN model using 1.5x of the original input size (multi-crop method).\n",
    "* Experiment #9 - Finetune the FCN model using 1.5x of the original input size (multi-crop method), also introducing a skip connection.   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test #1 - Finetune the original model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_load_img_type(0) \n",
    "model = setup_model()\n",
    "train_flow, test_flow = setup_data_flows(upscale_ratio=1.0)\n",
    "train_model(model, 'orig', train_flow, test_flow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# delete original model from memory\n",
    "del model\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test #2 - Finetune the FCN version using the original input size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "set_load_img_type(0)\n",
    "model_fcn = setup_model(fcn=True)\n",
    "train_flow, test_flow = setup_data_flows(upscale_ratio=1.0)\n",
    "train_model(model_fcn, 'fcn_10', train_flow, test_flow)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test #3 - Finetune the FCN model using 1.5x of the original input size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_load_img_type(0)\n",
    "model_fcn = setup_model(fcn=True)\n",
    "train_flow_15, test_flow_15 = setup_data_flows(upscale_ratio=1.5)\n",
    "train_model(model_fcn, 'fcn_15', train_flow_15, test_flow_15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test #4 - Finetune the FCN model using 2.0x of the original input size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "set_load_img_type(0)\n",
    "model_fcn = setup_model(fcn=True)\n",
    "train_flow_20, test_flow_20 = setup_data_flows(upscale_ratio=2.0)\n",
    "train_model(model_fcn, 'fcn_20', train_flow_20, test_flow_20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test #5 - Finetune the FCN model using 3.0x of the original input size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_load_img_type(0)\n",
    "model_fcn = setup_model(fcn=True)\n",
    "train_flow_30, test_flow_30 = setup_data_flows(upscale_ratio=3.0)\n",
    "train_model(model_fcn, 'fcn_30', train_flow_30, test_flow_30) # Not completed on Xception"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test #6 - Finetune the FCN model using 1.5x of the original input size (padding method)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_load_img_type(1) # 'load_img' now calls 'load_img_pad'\n",
    "model_fcn = setup_model(fcn=True)\n",
    "train_flow_15, test_flow_15 = setup_data_flows(upscale_ratio=1.5)\n",
    "train_model(model_fcn, 'fcn_15_pad', train_flow_15, test_flow_15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test #7 - Finetune the FCN model using 1.5x of the original input size (crop method)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_load_img_type(3) # 'load_img' now calls 'load_img_crop'\n",
    "model_fcn = setup_model(fcn=True)\n",
    "train_flow_15, test_flow_15 = setup_data_flows(upscale_ratio=1.5)\n",
    "train_model(model_fcn, 'fcn_15_crop', train_flow_15, test_flow_15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test #8 - Finetune the FCN model using 1.5x of the original input size (multi-crop method)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_load_img_type(4) # 'load_img' now calls 'load_img_multicrop'\n",
    "model_fcn = setup_model(fcn=True)\n",
    "train_flow_15, test_flow_15 = setup_data_flows(upscale_ratio=1.5)\n",
    "train_model(model_fcn, 'fcn_15_multicrop', train_flow_15, test_flow_15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test #9 - Create a copy of the FCN model with a skip conncetion and finetune the new  model using 1.5x of the original input size (multi-crop method)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('~~~ Setting up skip connection model ~~~')\n",
    "# ONLY implemented for VGG16\n",
    "\n",
    "# copy model\n",
    "print('    copying model...\\n')\n",
    "model_fcn = setup_model(fcn=True)\n",
    "model_fcn_skip = keras.models.clone_model(model_fcn)\n",
    "print(model_fcn.summary(line_length=112))\n",
    "\n",
    "# remove GlobalMaxPooling, predictions, fc2 and fc1)\n",
    "print('\\nRemoving last layer...')\n",
    "model_fcn_skip.layers.pop()\n",
    "model_fcn_skip.layers.pop()\n",
    "model_fcn_skip.layers.pop()\n",
    "model_fcn_skip.layers.pop()\n",
    "\n",
    "# add skip connnection\n",
    "print('\\nAdding skip connection...')\n",
    "b2_pool = model_fcn_skip.get_layer(\"block2_pool\").output\n",
    "b5_pool = model_fcn_skip.get_layer(\"block5_pool\").output\n",
    "print('    Adding Conv2d after \"block2_pool\" (\"skip_1\")...')\n",
    "x = keras.layers.Conv2D(128,(1,1),activation='relu',name='skip_1',padding='same')(b2_pool)\n",
    "print('    Adding MaxPooling2D after \"block2_pool\" (\"skip_2\")...')\n",
    "x = keras.layers.MaxPooling2D((65,65),strides=(2,2),name='skip_2')(x)\n",
    "print('    Adding Concatenate...')\n",
    "x = keras.layers.Concatenate(name='concat')([x, b5_pool])\n",
    "print('    Adding \"fc1\"...')\n",
    "x = keras.layers.Conv2D(4096,(7, 7),strides=(1,1),activation='relu',name='fc1',padding='valid')(x)\n",
    "print('    Adding \"fc2\"...')\n",
    "x = keras.layers.Conv2D(4096,(1, 1),strides=(1,1),activation='relu',name='fc2',padding='valid')(x)\n",
    "print('    Adding \"prediction\"...')\n",
    "x = keras.layers.Conv2D(2,(1, 1),strides=(1,1),activation='softmax',name='predictions',padding='valid')(x)\n",
    "\n",
    "# add dropout layer\n",
    "if DROPOUT_RATE>0.01:\n",
    "    print('    Adding Dropout (rate=.%3f) on FCN...' % DROPOUT_RATE) \n",
    "    x = keras.layers.Dropout(DROPOUT_RATE)(x)\n",
    "\n",
    "# add global max pooling layer\n",
    "print('    Adding GlobalMaxPooling2d...')\n",
    "x = GlobalMaxPooling2D()(x)\n",
    "\n",
    "# compile model\n",
    "model_fcn_skip = Model(model_fcn_skip.input, x, name='GlobalMaxPooledFCNmulticrop')\n",
    "print('    ~~~ Created new model: model_gmax15_2_multicrop (', model_fcn_skip.name, ')')\n",
    "\n",
    "print('\\nModel summary with skip connection...')\n",
    "print(model_fcn_skip.summary(line_length=112))\n",
    "print(\"\\n\")\n",
    "\n",
    "# reload weights\n",
    "model_fcn.save_weights('tmp_w')\n",
    "model_fcn_skip.load_weights('tmp_w', by_name=True, reshape=False, skip_mismatch=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_load_img_type(4)  # 'load_img' now calls 'load_img_multicrop'\n",
    "train_flow_15, test_flow_15 = setup_data_flows(upscale_ratio=1.5)\n",
    "train_model(model_fcn_skip, 'fcn_15_multicrop_skip', train_flow_15, test_flow_15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}