Keras_Advanced_Coding_Style.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# 計時器\n",
    "import time\n",
    "\n",
    "# 大畫家\n",
    "import matplotlib.patches as mpatches\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# 各種好用套件\n",
    "import os, random, sys, keras, math\n",
    "import numpy as np\n",
    "\n",
    "# k~k~k~~~~Keras core sturcture\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Activation, Input, Dense, Lambda, Dropout, concatenate\n",
    "from keras.initializers import Ones\n",
    "# concatenate 就是 merge\n",
    "\n",
    "# some functions\n",
    "from keras import initializers\n",
    "from keras.constraints import max_norm, non_neg, unit_norm\n",
    "from keras import backend as K\n",
    "from keras import metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Syntx to Neural Network    \n",
    "* Step-by-step syntax to Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Alt text](img/model_0.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Dense(3, input_shape=(2,)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Alt text](img/model_0_2x3.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Dense(2, kernel_constraint=unit_norm))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Alt text](img/model_0_2x3_3x2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Dense(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Alt text](img/model_0_2x3_3x2_2x3.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.add(Activation('sigmoid'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Alt text](img/model_0_2x3_3x2_2x3_activation.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 3)                 9         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 2)                 8         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 3)                 9         \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 3)                 0         \n",
      "=================================================================\n",
      "Total params: 26\n",
      "Trainable params: 26\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Re-write above NN using Model API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 2)                 0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 3)                 9         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 2)                 8         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 3)                 9         \n",
      "=================================================================\n",
      "Total params: 26\n",
      "Trainable params: 26\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "x = Input(shape=(2,))\n",
    "\n",
    "f = Dense(3, activation='relu')\n",
    "u = f(x)\n",
    "\n",
    "g = Dense(2, activation='relu')\n",
    "v = g(u)\n",
    "\n",
    "h = Dense(3, activation='sigmoid')\n",
    "y = h(v)\n",
    "\n",
    "model = Model(x, y)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 2)                 0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 3)                 9         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 2)                 8         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 3)                 9         \n",
      "=================================================================\n",
      "Total params: 26\n",
      "Trainable params: 26\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "x = Input(shape=(2,))\n",
    "u = Dense(3, activation='relu')(x)\n",
    "v = Dense(2, activation='relu')(u)\n",
    "y = Dense(3, activation='sigmoid')(v)\n",
    "\n",
    "model = Model(x, y)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 2)                 0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 2)                 6         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 3)                 9         \n",
      "=================================================================\n",
      "Total params: 15\n",
      "Trainable params: 15\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# ...... just for fun\n",
    "x = Input(shape=(2,))\n",
    "y = Dense(3, activation='sigmoid')(Dense(2, activation='relu')(x))\n",
    "\n",
    "model = Model(x, y)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Black Magic - Shared Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = Input(shape=(2,))\n",
    "\n",
    "f = Dense(3, activation='relu')\n",
    "u = f(x)\n",
    "\n",
    "g = Dense(2, activation='relu')\n",
    "v = g(u)\n",
    "\n",
    "h = Dense(3, activation='sigmoid')\n",
    "y = h(v)\n",
    "\n",
    "model = Model(x, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Black Magic - Branch and Merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = Input(shape=(2,), name='x_is_input')\n",
    "\n",
    "f = Dense(1, activation='relu', name='f')\n",
    "u_1 = f(x)\n",
    "\n",
    "g = Dense(2, activation='relu', name='g')\n",
    "u_2 = g(x)\n",
    "\n",
    "# merge layer\n",
    "merged_v = concatenate([u_1, u_2], name='merged_layer')\n",
    "\n",
    "h = Dense(1, activation='softmax', name='h')\n",
    "y = h(merged_v)\n",
    "\n",
    "# modeling\n",
    "model = Model(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "x_is_input (InputLayer)          (None, 2)             0                                            \n",
      "____________________________________________________________________________________________________\n",
      "f (Dense)                        (None, 1)             3           x_is_input[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "g (Dense)                        (None, 2)             6           x_is_input[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "merged_layer (Concatenate)       (None, 3)             0           f[0][0]                          \n",
      "                                                                   g[0][0]                          \n",
      "____________________________________________________________________________________________________\n",
      "h (Dense)                        (None, 1)             4           merged_layer[0][0]               \n",
      "====================================================================================================\n",
      "Total params: 13\n",
      "Trainable params: 13\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Black Magic - Branch and Merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = Input(shape=(2,), name='x_is_input')\n",
    "\n",
    "f = Dense(3, activation='relu', name='before_branch')\n",
    "u = f(x)\n",
    "\n",
    "# red route\n",
    "g_red = Dense(2, activation='relu',name='red')\n",
    "v_1 = g_red(u)\n",
    "\n",
    "# blue route\n",
    "g_blue_1 = Dense(1, activation='sigmoid',name='blue_1')\n",
    "v_blue = g_blue_1(u)\n",
    "\n",
    "g_blue_2 = Dense(2, activation='sigmoid',name='blue_2')\n",
    "v_2 = g_blue_2(v_blue)\n",
    "\n",
    "# merge layer\n",
    "merged_v = concatenate([v_1, v_2], name='merged_vector')\n",
    "\n",
    "h = Dense(3, activation='softmax', name='before_output')\n",
    "y = h(merged_v)\n",
    "\n",
    "# modeling\n",
    "model = Model(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "x_is_input (InputLayer)          (None, 2)             0                                            \n",
      "____________________________________________________________________________________________________\n",
      "before_branch (Dense)            (None, 3)             9           x_is_input[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "blue_1 (Dense)                   (None, 1)             4           before_branch[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "red (Dense)                      (None, 2)             8           before_branch[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "blue_2 (Dense)                   (None, 2)             4           blue_1[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "merged_vector (Concatenate)      (None, 4)             0           red[0][0]                        \n",
      "                                                                   blue_2[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "before_output (Dense)            (None, 3)             15          merged_vector[0][0]              \n",
      "====================================================================================================\n",
      "Total params: 40\n",
      "Trainable params: 40\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Black Magic - Customized Layer    \n",
    "* For more detai, please read: https://keras.io/layers/core/#lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def my_avergae(args):\n",
    "    return K.mean(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = Input(shape=(3,), name='x_is_input')\n",
    "\n",
    "f = Lambda(my_avergae, output_shape=(1,))\n",
    "y = f(x)\n",
    "\n",
    "model = Model(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "x_is_input (InputLayer)      (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "lambda_1 (Lambda)            (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 0\n",
      "Trainable params: 0\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3.], dtype=float32)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(np.array([[4,2,3]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Black Magic - Customized Loss Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = Input(shape=(2,))\n",
    "\n",
    "f = Dense(3, activation='relu')\n",
    "u = f(x)\n",
    "\n",
    "g = Dense(2, activation='relu')\n",
    "v = g(u)\n",
    "\n",
    "h = Dense(3, activation='sigmoid')\n",
    "y = h(v)\n",
    "\n",
    "model = Model(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# re-defined MSE by user\n",
    "def my_loss(y_true, y_pred):\n",
    "    error = y_true - y_pred\n",
    "    square_error = K.square(error)\n",
    "    mean_square_error = K.mean(square_error)\n",
    "    return mean_square_error\n",
    "\n",
    "def my_loss_2(y_true, y_pred):\n",
    "    return K.mean(K.square(y_true - y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss=my_loss, optimizer='SGD')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Black Magic Revisit – Submodel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "x_is_input (InputLayer)      (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "x_to_encoder (Dense)         (None, 100)               78500     \n",
      "_________________________________________________________________\n",
      "encoder_to_latent (Dense)    (None, 2)                 202       \n",
      "_________________________________________________________________\n",
      "latent_to_decoder (Dense)    (None, 100)               300       \n",
      "_________________________________________________________________\n",
      "deconder_to_x (Dense)        (None, 784)               79184     \n",
      "=================================================================\n",
      "Total params: 158,186\n",
      "Trainable params: 158,186\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "x = Input(shape=(28*28,), name='x_is_input')\n",
    "\n",
    "enc_1 = Dense(100, activation='relu', kernel_initializer=Ones(), name='x_to_encoder')\n",
    "x_enc = enc_1(x)\n",
    "\n",
    "enc_2 = Dense(2, activation='relu', kernel_initializer=Ones(),name='encoder_to_latent')\n",
    "latent = enc_2(x_enc)\n",
    "\n",
    "dec_2 = Dense(100, activation='relu', kernel_initializer=Ones(),name='latent_to_decoder')\n",
    "x_dec = dec_2(latent)\n",
    "\n",
    "dec_1 = Dense(28*28, activation='relu', kernel_initializer=Ones(),name='deconder_to_x')\n",
    "x_reconstruct = dec_1(x_dec)\n",
    "\n",
    "# modeling\n",
    "autoencoder = Model(x, x_reconstruct)\n",
    "autoencoder.compile(loss='mse', optimizer='SGD')\n",
    "autoencoder.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "x_is_input (InputLayer)      (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "x_to_encoder (Dense)         (None, 100)               78500     \n",
      "_________________________________________________________________\n",
      "encoder_to_latent (Dense)    (None, 2)                 202       \n",
      "=================================================================\n",
      "Total params: 78,702\n",
      "Trainable params: 78,702\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "latent_as_input (InputLayer) (None, 2)                 0         \n",
      "_________________________________________________________________\n",
      "latent_to_decoder (Dense)    (None, 100)               300       \n",
      "_________________________________________________________________\n",
      "deconder_to_x (Dense)        (None, 784)               79184     \n",
      "=================================================================\n",
      "Total params: 79,484\n",
      "Trainable params: 79,484\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "encoder = Model(x, latent)\n",
    "encoder.summary()\n",
    "\n",
    "decoder_input = Input(shape=(2,), name='latent_as_input')\n",
    "\n",
    "# dec_2 = Dense(100, activation='relu', name='latent_to_decoder')\n",
    "# x_dec = dec_2(latent)\n",
    "_dec = dec_2(decoder_input)\n",
    "\n",
    "# dec_1 = Dense(28*28, activation='relu', name='deconder_to_x')\n",
    "# x_reconstruct = dec_1(x_dec)\n",
    "_reconstruct = dec_1(_dec)\n",
    "\n",
    "\n",
    "decoder = Model(decoder_input, _reconstruct)\n",
    "decoder.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# single 28-by-28 matrix with all entry equals to 1\n",
    "x_test = np.array([np.ones(28*28)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "784"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# input x_test to see its reconstructed result\n",
    "ae_output = autoencoder.predict(x_test)\n",
    "\n",
    "# encode x_test to get encoded result\n",
    "# then decode it to see the reconstruct result\n",
    "x_test_encoded = encoder.predict(x_test)\n",
    "enc_dec_output = decoder.predict(x_test_encoded)\n",
    "\n",
    "# check if two results are the same\n",
    "(ae_output == enc_dec_output).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Assignment Revisit - Variational Autoencoder    \n",
    "* Understand how VAE works\n",
    "* Define what we need to build VAE\n",
    "* Review the code of VAE!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}