Build AlexNet and test on CIFAR10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's construct AlexNet in Keras!\n",
    "\n",
    "#### First let's load and prep our CIFAR10 data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.datasets import cifar10\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.regularizers import l2\n",
    "\n",
    "# Loads the CIFAR dataset\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "\n",
    "# Display our data shape/dimensions\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# Now we one hot encode outputs\n",
    "num_classes = 10\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now let's create our layers to replicate AlexNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 32, 32, 96)        34944     \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 32, 32, 96)        384       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 32, 32, 96)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 16, 16, 96)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 16, 16, 256)       614656    \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 16, 16, 256)       1024      \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 16, 16, 256)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 8, 8, 256)         0         \n",
      "_________________________________________________________________\n",
      "zero_padding2d_1 (ZeroPaddin (None, 10, 10, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 10, 10, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 10, 10, 512)       2048      \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 10, 10, 512)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 5, 5, 512)         0         \n",
      "_________________________________________________________________\n",
      "zero_padding2d_2 (ZeroPaddin (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 7, 7, 1024)        4719616   \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 7, 7, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 7, 7, 1024)        0         \n",
      "_________________________________________________________________\n",
      "zero_padding2d_3 (ZeroPaddin (None, 9, 9, 1024)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, 9, 9, 1024)        9438208   \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 9, 9, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 9, 9, 1024)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 4, 4, 1024)        0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 16384)             0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 3072)              50334720  \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 3072)              12288     \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 3072)              0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 3072)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 4096)              12587008  \n",
      "_________________________________________________________________\n",
      "batch_normalization_7 (Batch (None, 4096)              16384     \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                40970     \n",
      "_________________________________________________________________\n",
      "batch_normalization_8 (Batch (None, 10)                40        \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 78,990,642\n",
      "Trainable params: 78,970,462\n",
      "Non-trainable params: 20,180\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "l2_reg = 0\n",
    "\n",
    "# Initialize model\n",
    "model = Sequential()\n",
    "\n",
    "# 1st Conv Layer \n",
    "model.add(Conv2D(96, (11, 11), input_shape=x_train.shape[1:],\n",
    "    padding='same', kernel_regularizer=l2(l2_reg)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "# 2nd Conv Layer \n",
    "model.add(Conv2D(256, (5, 5), padding='same'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "# 3rd Conv Layer \n",
    "model.add(ZeroPadding2D((1, 1)))\n",
    "model.add(Conv2D(512, (3, 3), padding='same'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "# 4th Conv Layer \n",
    "model.add(ZeroPadding2D((1, 1)))\n",
    "model.add(Conv2D(1024, (3, 3), padding='same'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation('relu'))\n",
    "\n",
    "# 5th Conv Layer \n",
    "model.add(ZeroPadding2D((1, 1)))\n",
    "model.add(Conv2D(1024, (3, 3), padding='same'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\n",
    "# 1st FC Layer\n",
    "model.add(Flatten())\n",
    "model.add(Dense(3072))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "\n",
    "# 2nd FC Layer\n",
    "model.add(Dense(4096))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "\n",
    "# 3rd FC Layer\n",
    "model.add(Dense(num_classes))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "print(model.summary())\n",
    "\n",
    "model.compile(loss = 'categorical_crossentropy',\n",
    "              optimizer = keras.optimizers.Adadelta(),\n",
    "              metrics = ['accuracy'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now let us train AlexNet on our CIFAR10 Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      "  608/50000 [..............................] - ETA: 2:45:39 - loss: 2.2872 - accuracy: 0.1793"
     ]
    }
   ],
   "source": [
    "# Training Parameters\n",
    "batch_size = 32\n",
    "epochs = 1\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          validation_data=(x_test, y_test),\n",
    "          shuffle=True)\n",
    "\n",
    "model.save(\"CIFAR10_AlexNet_1_Epoch.h5\")\n",
    "\n",
    "# Evaluate the performance of our trained model\n",
    "scores = model.evaluate(x_test, y_test, verbose=1)\n",
    "print('Test loss:', scores[0])\n",
    "print('Test accuracy:', scores[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}