1_Introduction_to_Tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nWhat is Tensorflow?\\n\\nIt's an open source Deep Learning library released by Google. It provides the ability to define functions on tensors.\\nA tensor is a multilinear map from vector spaces to the real number. It's a multidimensional array of numbers.\\n\\n\""
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "What is Tensorflow?\n",
    "\n",
    "It's an open source Deep Learning library released by Google. It provides the ability to define functions on tensors.\n",
    "A tensor is a multilinear map from vector spaces to the real number. It's a multidimensional array of numbers.\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"Resources/np_to_tf.png\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tensorflow is also similar to numpy, although the latter has no GPU support\n",
    "# tensorflow offers all the computations numpy does\n",
    "from IPython.display import Image\n",
    "Image(url='Resources/np_to_tf.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's import tensorflow as tf\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's look at tensorflow data types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# constants in tensorflow are variables that contain fixed values given on their creation\n",
    "V1 = tf.constant([1.,2.])\n",
    "# you can also pass explicit names, just like in theano\n",
    "V2 = tf.constant([3.,4.], name='const_V2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# name arguments are useful if you want to save your variables to a binary file\n",
    "# they are also used in TensorBoard, a visual representation of Tensorflow calculations (we will get to it later)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Const:0\", shape=(2,), dtype=float32)\n",
      "Tensor(\"const_V2:0\", shape=(2,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# these types produce tensors, tensor is a typed multy-dimensional array\n",
    "print(V1)\n",
    "print(V2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"add:0\", shape=(2,), dtype=float32)\n",
      "Tensor(\"mul:0\", shape=(2,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# you can then conduct various operations on your tensors\n",
    "V3 = V1 + V2\n",
    "V4 = V1 * V2\n",
    "print(V3)\n",
    "print(V4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we created a computational graph, it won't be calculated until we tell tensorflow to evaluate it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Cannot evaluate tensor using `eval()`: No default session is registered. Use `with sess.as_default()` or pass an explicit session to `eval(session=sess)`",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-3bb9a6cfdb9a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# we would need to call .eval() first\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mV3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mV4\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mj:\\anaconda_python3_6\\envs\\cuda\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36meval\u001b[1;34m(self, feed_dict, session)\u001b[0m\n\u001b[0;32m    646\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    647\u001b[0m     \"\"\"\n\u001b[1;32m--> 648\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_eval_using_default_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    649\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    650\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mj:\\anaconda_python3_6\\envs\\cuda\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36m_eval_using_default_session\u001b[1;34m(tensors, feed_dict, graph, session)\u001b[0m\n\u001b[0;32m   4742\u001b[0m     \u001b[0msession\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_default_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4743\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0msession\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4744\u001b[1;33m       raise ValueError(\"Cannot evaluate tensor using `eval()`: No default \"\n\u001b[0m\u001b[0;32m   4745\u001b[0m                        \u001b[1;34m\"session is registered. Use `with \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4746\u001b[0m                        \u001b[1;34m\"sess.as_default()` or pass an explicit session to \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Cannot evaluate tensor using `eval()`: No default session is registered. Use `with sess.as_default()` or pass an explicit session to `eval(session=sess)`"
     ]
    }
   ],
   "source": [
    "# we would need to call .eval() first\n",
    "V3.eval()\n",
    "V4.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# that didn't work! but it's fine, we missed one important step\n",
    "# all operations in tensorflow are evaluated within sessions\n",
    "# let's create a session first\n",
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"add_1:0\", shape=(2,), dtype=float32)\n",
      "Tensor(\"mul_1:0\", shape=(2,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# let's create our graphs\n",
    "V3 = V1 + V2\n",
    "print(V3)\n",
    "V4 = V1 * V2\n",
    "print(V4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4., 6.], dtype=float32)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now we will pass the session to the eval() fcuntion\n",
    "V3.eval(session=sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3., 8.], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "V4.eval(session=sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3. 8.]\n"
     ]
    }
   ],
   "source": [
    "# another way of doing the same action is by running sess.run() on the created graph\n",
    "print(sess.run(V4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# close the session\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# that looks tedious, but there is a better way of doing this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# interactive sessions can save you a lot of time\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "V3 = V1 + V2\n",
    "V4 = V1 * V2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4., 6.], dtype=float32)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now we don't need to care about passing sessions\n",
    "V3.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3., 8.], dtype=float32)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "V4.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# working with placeholders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# basically, we define the size and what the input type is\n",
    "# at runtime we initialize placeholders with data\n",
    "x = tf.placeholder(tf.float32, shape=(1024, 1024))\n",
    "y = tf.matmul(x, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a random array\n",
    "import numpy as np\n",
    "rand_array = np.random.rand(1024, 1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[258.39197 253.94879 251.3118  ... 254.28647 247.59923 259.94205]\n",
      " [263.3557  256.33743 258.41504 ... 259.75125 256.5243  266.22382]\n",
      " [261.2799  257.82227 257.84375 ... 255.47865 256.51367 260.3273 ]\n",
      " ...\n",
      " [256.66608 247.51122 250.21756 ... 258.41742 252.0644  253.21973]\n",
      " [257.13403 247.57468 251.75722 ... 252.73485 243.71902 250.30785]\n",
      " [257.8908  241.81041 247.07864 ... 255.5135  252.0509  252.52078]]\n"
     ]
    }
   ],
   "source": [
    "# feed x with values\n",
    "# this can only be done within sess.run(), the values are fed with \"feed_dict\" argument\n",
    "print(sess.run(y, feed_dict={x: rand_array}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# working with variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# another data type in Tensorflow in tf.Variable\n",
    "# these are not constant, the values can be changed using .assign() method\n",
    "w = tf.Variable([1.,3.], name='variable_w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Assign:0' shape=(2,) dtype=float32_ref>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w.assign(w + 1.1)  # or w.assign_add(.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# now how do I get the value of w?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we would need to initialize our w Variable for it to be used in our computations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 3.]\n"
     ]
    }
   ],
   "source": [
    "# let's launch the graph in a session\n",
    "with tf.Session() as sess:\n",
    "    sess.run(w.initializer)\n",
    "    print(w.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 3.]\n",
      "[1.1 3.1]\n",
      "[1.1 3.1]\n"
     ]
    }
   ],
   "source": [
    "# our w wasn't changed even though we called .assign() on it\n",
    "# it needs to be initialized before we assign new values to it\n",
    "with tf.Session() as sess:\n",
    "    sess.run(w.initializer)\n",
    "    print(w.eval())\n",
    "    print(w.assign(w + .1).eval())\n",
    "    print(w.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# what if we have multiple variables, do we have to initialize each of them?\n",
    "w = tf.Variable([1.,3.], name='variable_1')\n",
    "w2 = tf.Variable([2.,4.], name='variable_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 3.]\n",
      "[2. 4.]\n"
     ]
    }
   ],
   "source": [
    "# of course not, we can add an operation to initialize all variables we might have\n",
    "# by using tf.global_variables_initializer()\n",
    "init_op = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init_op)\n",
    "    print(w.eval())\n",
    "    print(w2.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we created several session and conducted some computations\n",
    "# tensorflow saved all of them so that we can debug them\n",
    "# for now, let's get rid of them\n",
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's start a new session\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python tensorflow, has all the same caveats Python has\n",
    "a = tf.constant(23)\n",
    "b = tf.constant(11)\n",
    "c = tf.div(a,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# what will this evaluate to?\n",
    "c.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.090909090909091"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use truediv instead\n",
    "c = tf.truediv(a,b)\n",
    "c.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The computations in Tensorflow are saved as Directed graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"https://www.tensorflow.org/images/tensors_flowing.gif\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import Image\n",
    "Image(url='https://www.tensorflow.org/images/tensors_flowing.gif')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to visualize your graphs, you can use TensorBoard\n",
    "# the name arguments are used here to help you identify your variables\n",
    "# and this helps you take a more detailed look into the computations your Deep Learning model is going through"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"https://www.tensorflow.org/images/graph_vis_animation.gif\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import Image\n",
    "Image(url='https://www.tensorflow.org/images/graph_vis_animation.gif')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to start your TensorBoard, you would first need to dump your data to disk\n",
    "writer = tf.summary.FileWriter(\"/tmp\")\n",
    "writer.add_graph(sess.graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# then run tensorboard from the terminal\n",
    "# tensorboard --logdir /tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# but why not visualize it directly in our Jupyter Notebook?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's use this hacky way to do it!\n",
    "# courtesy of Deepdream \n",
    "# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/deepdream/deepdream.ipynb\n",
    "\n",
    "from IPython.display import clear_output, Image, display, HTML\n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = \"<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "def show_graph(graph_def, max_const_size=32):\n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.08632412485326468&quot;).pbtxt = 'node {\\n  name: &quot;Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 23\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 11\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;div&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;Const&quot;\\n  input: &quot;Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;truediv/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;Const&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_DOUBLE\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;truediv/Cast_1&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;Const_1&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_DOUBLE\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;truediv/Cast&quot;\\n  input: &quot;truediv/Cast_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_DOUBLE\\n    }\\n  }\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.08632412485326468&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# let's see what graphs we have logged today so far\n",
    "# our hidden scope has been created and we can open it to look what's inside\n",
    "show_graph(tf.get_default_graph().as_graph_def())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n",
      "[[0.13357091 0.5811863 ]]\n",
      "[0.]\n"
     ]
    }
   ],
   "source": [
    "# you can define your values within scopes, to help declutter the visualization\n",
    "# in TensorBoard all underlying variables will be collapsed under 'hidden' node label\n",
    "with tf.name_scope('hidden') as scope:\n",
    "    a = tf.constant(5, name='alpha')\n",
    "    W = tf.Variable(tf.random_uniform([1, 2], -1.0, 1.0), name='weights')\n",
    "    b = tf.Variable(tf.zeros([1]), name='biases')\n",
    "    \n",
    "init_op = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init_op)\n",
    "    \n",
    "    print(a.eval())\n",
    "    print(W.eval())\n",
    "    print(b.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.17075859529416593&quot;).pbtxt = 'node {\\n  name: &quot;hidden/alpha&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\001\\\\000\\\\000\\\\000\\\\002\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden/random_uniform/max&quot;\\n  input: &quot;hidden/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden/random_uniform/mul&quot;\\n  input: &quot;hidden/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/weights&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 1\\n        }\\n        dim {\\n          size: 2\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/weights/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden/weights&quot;\\n  input: &quot;hidden/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/weights/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden/weights&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/biases&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 1\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/biases/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden/biases&quot;\\n  input: &quot;hidden/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden/biases/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden/biases&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden/weights/Assign&quot;\\n  input: &quot;^hidden/biases/Assign&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.17075859529416593&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# to start your TensorBoard, you would first need to dump your data to disk\n",
    "writer = tf.summary.FileWriter(\"/tmp\")\n",
    "writer.add_graph(sess.graph)\n",
    "show_graph(tf.get_default_graph().as_graph_def())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import MNIST data from Yann LeCun\n",
    "\"\"\"\n",
    "MNIST is a simple computer vision dataset. It consists of images of handwritten digits, it includes labels \n",
    "for each image, telling us which digit it is\n",
    "\n",
    "55,000 data points of training data (mnist.train), \n",
    "10,000 points of test data (mnist.test), and \n",
    "5,000 points of validation data (mnist.validation)\n",
    "\"\"\"\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADepJREFUeJzt3X+MVfWZx/HPI1v8AYTAMo7Ewk4xk03UuJTcoFiy6cZt\ntaYJ1hitJAQTA8a0TRtLUmVJ1viHmWwWGxI3jXQlBcNKNwKBGNNVyEYkWRuuiIrgLmqmAeTHgCYV\n+YMyffaPOTSjzvne6z3n3nNnnvcrmcy95zk/nhz9cO693zvna+4uAPFcVnUDAKpB+IGgCD8QFOEH\ngiL8QFCEHwiK8ANBEX4gKMIPBPVXnTzYrFmzvK+vr5OHBEIZHBzUmTNnrJl1C4XfzO6QtE7SJEn/\n7u4DqfX7+vpUr9eLHBJAQq1Wa3rdll/2m9kkSf8m6XuSrpd0v5ld3+r+AHRWkff8CyW97+4fuvsF\nSVskLSmnLQDtViT810o6Our5sWzZ55jZSjOrm1l9aGiowOEAlKntn/a7+3p3r7l7raenp92HA9Ck\nIuE/LmnOqOdfz5YBGAeKhH+fpH4z+4aZTZb0Q0k7y2kLQLu1PNTn7hfN7MeS/ksjQ30b3P3d0joD\n0FaFxvnd/SVJL5XUC4AO4uu9QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ER\nfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANB\nEX4gKMIPBFVoll4zG5T0qaRhSRfdvVZGUwDar1D4M//g7mdK2A+ADuJlPxBU0fC7pF1m9oaZrSyj\nIQCdUfRl/2J3P25mV0t6xczec/c9o1fI/lFYKUlz584teDgAZSl05Xf349nv05K2S1o4xjrr3b3m\n7rWenp4ihwNQopbDb2ZTzGzapceSvivpYFmNAWivIi/7eyVtN7NL+/kPd/9dKV0BaLuWw+/uH0r6\nuxJ7AdBBDPUBQRF+ICjCDwRF+IGgCD8QFOEHgirjr/pQsV27duXWsu9h5JoxY0ayfvBg+ntbixYt\nStb7+/uTdVSHKz8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBDVhxvn37NmTrL/++uvJ+tq1a8tsp6PO\nnj3b8raTJk1K1i9cuJCsX3XVVcn61KlTc2uLFy9Obvvcc88VOjbSuPIDQRF+ICjCDwRF+IGgCD8Q\nFOEHgiL8QFDjapx/YGAgt7ZmzZrktsPDw2W3MyEUPS/nz59vub5t27bkto3uRbBx48ZkfcqUKcl6\ndFz5gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiCohuP8ZrZB0vclnXb3G7NlMyX9VlKfpEFJ97r7J+1r\nc8QzzzyTW2s0Xn3LLbck69OmTWuppzLcdtttyfrdd9/doU6+updffjlZX7duXW7tyJEjyW23bt3a\nUk+XbNq0KbfGvQCau/L/RtIdX1j2qKTd7t4vaXf2HMA40jD87r5H0sdfWLxE0qWvV22UdFfJfQFo\ns1bf8/e6+4ns8UlJvSX1A6BDCn/g5+4uyfPqZrbSzOpmVh8aGip6OAAlaTX8p8xstiRlv0/nreju\n69295u61np6eFg8HoGythn+npOXZ4+WSdpTTDoBOaRh+M3te0v9I+lszO2ZmD0oakPQdMzsi6R+z\n5wDGERt5y94ZtVrN6/V6y9ufOXMmt/bBBx8kt50/f36yfvnll7fUE9I++ST/6x+Nvt/w5ptvFjr2\n5s2bc2tLly4ttO9uVavVVK/X0zdCyPANPyAowg8ERfiBoAg/EBThB4Ii/EBQ42qoDxNLo2nTFy1a\nVGj/vb35f3Jy8uTJQvvuVgz1AWiI8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAo\nwg8ERfiBoAg/EBThB4JqOEU3UMSOHfnzuezdu7etx/7ss89ya0ePHk1uO2fOnLLb6Tpc+YGgCD8Q\nFOEHgiL8QFCEHwiK8ANBEX4gqIbj/Ga2QdL3JZ129xuzZY9LWiFpKFtttbu/1K4mkXbu3Lnc2vbt\n25Pbrlmzpux2Pic1nt7uOSNS5+Wmm25KbpuaWnyiaObK/xtJd4yx/JfuPj/7IfjAONMw/O6+R9LH\nHegFQAcVec//EzN728w2mNmM0joC0BGthv9XkuZJmi/phKS1eSua2Uozq5tZfWhoKG81AB3WUvjd\n/ZS7D7v7nyX9WtLCxLrr3b3m7rWenp5W+wRQspbCb2azRz39gaSD5bQDoFOaGep7XtK3Jc0ys2OS\n/lnSt81sviSXNCjpoTb2CKANGobf3e8fY/GzbeglrEOHDiXr+/btS9YHBgZya++9915LPU10q1at\nqrqFyvENPyAowg8ERfiBoAg/EBThB4Ii/EBQ3Lq7BGfPnk3WH3744WT9hRdeSNbb+aev1113XbJ+\nzTXXFNr/008/nVubPHlyctulS5cm62+99VZLPUnS3LlzW952ouDKDwRF+IGgCD8QFOEHgiL8QFCE\nHwiK8ANBMc7fpC1btuTWnnjiieS2hw8fTtanTZuWrM+cOTNZf/LJJ3NrjaaabnQL6+nTpyfr7VT0\nzk+p3m+//fZC+54IuPIDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM8zfp1Vdfza01Gsd/4IEHkvXV\nq1cn6/39/cn6eHX8+PFkvdEtzRu54oorcmtXX311oX1PBFz5gaAIPxAU4QeCIvxAUIQfCIrwA0ER\nfiCohuP8ZjZH0iZJvZJc0np3X2dmMyX9VlKfpEFJ97r7J+1rtVpPPfVUbm3BggXJbVesWFF2OxPC\n0aNHk/WPPvqo0P7vueeeQttPdM1c+S9K+rm7Xy/pFkk/MrPrJT0qabe790vanT0HME40DL+7n3D3\n/dnjTyUdlnStpCWSNmarbZR0V7uaBFC+r/Se38z6JH1T0u8l9br7iax0UiNvCwCME02H38ymStoq\n6Wfu/sfRNR+ZTG7MCeXMbKWZ1c2sPjQ0VKhZAOVpKvxm9jWNBH+zu2/LFp8ys9lZfbak02Nt6+7r\n3b3m7rWiN2QEUJ6G4Tczk/SspMPuPvoj752SlmePl0vaUX57ANqlmT/p/ZakZZLeMbMD2bLVkgYk\n/aeZPSjpD5LubU+L3eHKK6/MrTGU15rUn0k3o9EtzR955JFC+5/oGobf3fdKspzybeW2A6BT+IYf\nEBThB4Ii/EBQhB8IivADQRF+IChu3Y22uvnmm3Nr+/fvL7Tv++67L1mfN29eof1PdFz5gaAIPxAU\n4QeCIvxAUIQfCIrwA0ERfiAoxvnRVqnpyy9evJjcdsaMGcn6qlWrWuoJI7jyA0ERfiAowg8ERfiB\noAg/EBThB4Ii/EBQjPOjkNdeey1ZP3/+fG5t+vTpyW1ffPHFZJ2/1y+GKz8QFOEHgiL8QFCEHwiK\n8ANBEX4gKMIPBNVwnN/M5kjaJKlXkkta7+7rzOxxSSskDWWrrnb3l9rVKKoxPDycrD/22GPJ+uTJ\nk3NrK1asSG576623Jusoppkv+VyU9HN3329m0yS9YWavZLVfuvu/tq89AO3SMPzufkLSiezxp2Z2\nWNK17W4MQHt9pff8ZtYn6ZuSfp8t+omZvW1mG8xszHsumdlKM6ubWX1oaGisVQBUoOnwm9lUSVsl\n/czd/yjpV5LmSZqvkVcGa8fazt3Xu3vN3Ws9PT0ltAygDE2F38y+ppHgb3b3bZLk7qfcfdjd/yzp\n15IWtq9NAGVrGH4zM0nPSjrs7k+NWj571Go/kHSw/PYAtEszn/Z/S9IySe+Y2YFs2WpJ95vZfI0M\n/w1KeqgtHaJSI//253voofR/9gULFuTWbrjhhpZ6Qjma+bR/r6Sx/g9gTB8Yx/iGHxAU4QeCIvxA\nUIQfCIrwA0ERfiAobt2NpMsuS18fli1b1qFOUDau/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QlLl7\n5w5mNiTpD6MWzZJ0pmMNfDXd2lu39iXRW6vK7O1v3L2p++V1NPxfOrhZ3d1rlTWQ0K29dWtfEr21\nqqreeNkPBEX4gaCqDv/6io+f0q29dWtfEr21qpLeKn3PD6A6VV/5AVSkkvCb2R1m9r9m9r6ZPVpF\nD3nMbNDM3jGzA2ZWr7iXDWZ22swOjlo208xeMbMj2e8xp0mrqLfHzex4du4OmNmdFfU2x8z+28wO\nmdm7ZvbTbHml5y7RVyXnreMv+81skqT/k/QdScck7ZN0v7sf6mgjOcxsUFLN3SsfEzazv5d0TtIm\nd78xW/Yvkj5294HsH84Z7v6LLuntcUnnqp65OZtQZvbomaUl3SXpAVV47hJ93asKzlsVV/6Fkt53\n9w/d/YKkLZKWVNBH13P3PZI+/sLiJZI2Zo83auR/no7L6a0ruPsJd9+fPf5U0qWZpSs9d4m+KlFF\n+K+VdHTU82Pqrim/XdIuM3vDzFZW3cwYerNp0yXppKTeKpsZQ8OZmzvpCzNLd825a2XG67Lxgd+X\nLXb3+ZK+J+lH2cvbruQj79m6abimqZmbO2WMmaX/ospz1+qM12WrIvzHJc0Z9fzr2bKu4O7Hs9+n\nJW1X980+fOrSJKnZ79MV9/MX3TRz81gzS6sLzl03zXhdRfj3Seo3s2+Y2WRJP5S0s4I+vsTMpmQf\nxMjMpkj6rrpv9uGdkpZnj5dL2lFhL5/TLTM3580srYrPXdfNeO3uHf+RdKdGPvH/QNI/VdFDTl/z\nJL2V/bxbdW+SntfIy8A/aeSzkQcl/bWk3ZKOSNolaWYX9facpHckva2RoM2uqLfFGnlJ/7akA9nP\nnVWfu0RflZw3vuEHBMUHfkBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgvp/UqBHBigpANMAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1bbe3bfecc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# let's have a look at the data\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib\n",
    "%matplotlib inline\n",
    "\n",
    "tmp = mnist.train.images[0]\n",
    "tmp = tmp.reshape((28,28))\n",
    "\n",
    "plt.imshow(tmp, cmap = cm.Greys)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    }
   ],
   "source": [
    "print(mnist.train.labels[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADepJREFUeJzt3X+MVfWZx/HPI1v8AYTAMo7Ewk4xk03UuJTcoFiy6cZt\ntaYJ1hitJAQTA8a0TRtLUmVJ1viHmWwWGxI3jXQlBcNKNwKBGNNVyEYkWRuuiIrgLmqmAeTHgCYV\n+YMyffaPOTSjzvne6z3n3nNnnvcrmcy95zk/nhz9cO693zvna+4uAPFcVnUDAKpB+IGgCD8QFOEH\ngiL8QFCEHwiK8ANBEX4gKMIPBPVXnTzYrFmzvK+vr5OHBEIZHBzUmTNnrJl1C4XfzO6QtE7SJEn/\n7u4DqfX7+vpUr9eLHBJAQq1Wa3rdll/2m9kkSf8m6XuSrpd0v5ld3+r+AHRWkff8CyW97+4fuvsF\nSVskLSmnLQDtViT810o6Our5sWzZ55jZSjOrm1l9aGiowOEAlKntn/a7+3p3r7l7raenp92HA9Ck\nIuE/LmnOqOdfz5YBGAeKhH+fpH4z+4aZTZb0Q0k7y2kLQLu1PNTn7hfN7MeS/ksjQ30b3P3d0joD\n0FaFxvnd/SVJL5XUC4AO4uu9QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ER\nfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANB\nEX4gKMIPBFVoll4zG5T0qaRhSRfdvVZGUwDar1D4M//g7mdK2A+ADuJlPxBU0fC7pF1m9oaZrSyj\nIQCdUfRl/2J3P25mV0t6xczec/c9o1fI/lFYKUlz584teDgAZSl05Xf349nv05K2S1o4xjrr3b3m\n7rWenp4ihwNQopbDb2ZTzGzapceSvivpYFmNAWivIi/7eyVtN7NL+/kPd/9dKV0BaLuWw+/uH0r6\nuxJ7AdBBDPUBQRF+ICjCDwRF+IGgCD8QFOEHgirjr/pQsV27duXWsu9h5JoxY0ayfvBg+ntbixYt\nStb7+/uTdVSHKz8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBDVhxvn37NmTrL/++uvJ+tq1a8tsp6PO\nnj3b8raTJk1K1i9cuJCsX3XVVcn61KlTc2uLFy9Obvvcc88VOjbSuPIDQRF+ICjCDwRF+IGgCD8Q\nFOEHgiL8QFDjapx/YGAgt7ZmzZrktsPDw2W3MyEUPS/nz59vub5t27bkto3uRbBx48ZkfcqUKcl6\ndFz5gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiCohuP8ZrZB0vclnXb3G7NlMyX9VlKfpEFJ97r7J+1r\nc8QzzzyTW2s0Xn3LLbck69OmTWuppzLcdtttyfrdd9/doU6+updffjlZX7duXW7tyJEjyW23bt3a\nUk+XbNq0KbfGvQCau/L/RtIdX1j2qKTd7t4vaXf2HMA40jD87r5H0sdfWLxE0qWvV22UdFfJfQFo\ns1bf8/e6+4ns8UlJvSX1A6BDCn/g5+4uyfPqZrbSzOpmVh8aGip6OAAlaTX8p8xstiRlv0/nreju\n69295u61np6eFg8HoGythn+npOXZ4+WSdpTTDoBOaRh+M3te0v9I+lszO2ZmD0oakPQdMzsi6R+z\n5wDGERt5y94ZtVrN6/V6y9ufOXMmt/bBBx8kt50/f36yfvnll7fUE9I++ST/6x+Nvt/w5ptvFjr2\n5s2bc2tLly4ttO9uVavVVK/X0zdCyPANPyAowg8ERfiBoAg/EBThB4Ii/EBQ42qoDxNLo2nTFy1a\nVGj/vb35f3Jy8uTJQvvuVgz1AWiI8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAo\nwg8ERfiBoAg/EBThB4JqOEU3UMSOHfnzuezdu7etx/7ss89ya0ePHk1uO2fOnLLb6Tpc+YGgCD8Q\nFOEHgiL8QFCEHwiK8ANBEX4gqIbj/Ga2QdL3JZ129xuzZY9LWiFpKFtttbu/1K4mkXbu3Lnc2vbt\n25Pbrlmzpux2Pic1nt7uOSNS5+Wmm25KbpuaWnyiaObK/xtJd4yx/JfuPj/7IfjAONMw/O6+R9LH\nHegFQAcVec//EzN728w2mNmM0joC0BGthv9XkuZJmi/phKS1eSua2Uozq5tZfWhoKG81AB3WUvjd\n/ZS7D7v7nyX9WtLCxLrr3b3m7rWenp5W+wRQspbCb2azRz39gaSD5bQDoFOaGep7XtK3Jc0ys2OS\n/lnSt81sviSXNCjpoTb2CKANGobf3e8fY/GzbeglrEOHDiXr+/btS9YHBgZya++9915LPU10q1at\nqrqFyvENPyAowg8ERfiBoAg/EBThB4Ii/EBQ3Lq7BGfPnk3WH3744WT9hRdeSNbb+aev1113XbJ+\nzTXXFNr/008/nVubPHlyctulS5cm62+99VZLPUnS3LlzW952ouDKDwRF+IGgCD8QFOEHgiL8QFCE\nHwiK8ANBMc7fpC1btuTWnnjiieS2hw8fTtanTZuWrM+cOTNZf/LJJ3NrjaaabnQL6+nTpyfr7VT0\nzk+p3m+//fZC+54IuPIDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM8zfp1Vdfza01Gsd/4IEHkvXV\nq1cn6/39/cn6eHX8+PFkvdEtzRu54oorcmtXX311oX1PBFz5gaAIPxAU4QeCIvxAUIQfCIrwA0ER\nfiCohuP8ZjZH0iZJvZJc0np3X2dmMyX9VlKfpEFJ97r7J+1rtVpPPfVUbm3BggXJbVesWFF2OxPC\n0aNHk/WPPvqo0P7vueeeQttPdM1c+S9K+rm7Xy/pFkk/MrPrJT0qabe790vanT0HME40DL+7n3D3\n/dnjTyUdlnStpCWSNmarbZR0V7uaBFC+r/Se38z6JH1T0u8l9br7iax0UiNvCwCME02H38ymStoq\n6Wfu/sfRNR+ZTG7MCeXMbKWZ1c2sPjQ0VKhZAOVpKvxm9jWNBH+zu2/LFp8ys9lZfbak02Nt6+7r\n3b3m7rWiN2QEUJ6G4Tczk/SspMPuPvoj752SlmePl0vaUX57ANqlmT/p/ZakZZLeMbMD2bLVkgYk\n/aeZPSjpD5LubU+L3eHKK6/MrTGU15rUn0k3o9EtzR955JFC+5/oGobf3fdKspzybeW2A6BT+IYf\nEBThB4Ii/EBQhB8IivADQRF+IChu3Y22uvnmm3Nr+/fvL7Tv++67L1mfN29eof1PdFz5gaAIPxAU\n4QeCIvxAUIQfCIrwA0ERfiAoxvnRVqnpyy9evJjcdsaMGcn6qlWrWuoJI7jyA0ERfiAowg8ERfiB\noAg/EBThB4Ii/EBQjPOjkNdeey1ZP3/+fG5t+vTpyW1ffPHFZJ2/1y+GKz8QFOEHgiL8QFCEHwiK\n8ANBEX4gKMIPBNVwnN/M5kjaJKlXkkta7+7rzOxxSSskDWWrrnb3l9rVKKoxPDycrD/22GPJ+uTJ\nk3NrK1asSG576623Jusoppkv+VyU9HN3329m0yS9YWavZLVfuvu/tq89AO3SMPzufkLSiezxp2Z2\nWNK17W4MQHt9pff8ZtYn6ZuSfp8t+omZvW1mG8xszHsumdlKM6ubWX1oaGisVQBUoOnwm9lUSVsl\n/czd/yjpV5LmSZqvkVcGa8fazt3Xu3vN3Ws9PT0ltAygDE2F38y+ppHgb3b3bZLk7qfcfdjd/yzp\n15IWtq9NAGVrGH4zM0nPSjrs7k+NWj571Go/kHSw/PYAtEszn/Z/S9IySe+Y2YFs2WpJ95vZfI0M\n/w1KeqgtHaJSI//253voofR/9gULFuTWbrjhhpZ6Qjma+bR/r6Sx/g9gTB8Yx/iGHxAU4QeCIvxA\nUIQfCIrwA0ERfiAobt2NpMsuS18fli1b1qFOUDau/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QlLl7\n5w5mNiTpD6MWzZJ0pmMNfDXd2lu39iXRW6vK7O1v3L2p++V1NPxfOrhZ3d1rlTWQ0K29dWtfEr21\nqqreeNkPBEX4gaCqDv/6io+f0q29dWtfEr21qpLeKn3PD6A6VV/5AVSkkvCb2R1m9r9m9r6ZPVpF\nD3nMbNDM3jGzA2ZWr7iXDWZ22swOjlo208xeMbMj2e8xp0mrqLfHzex4du4OmNmdFfU2x8z+28wO\nmdm7ZvbTbHml5y7RVyXnreMv+81skqT/k/QdScck7ZN0v7sf6mgjOcxsUFLN3SsfEzazv5d0TtIm\nd78xW/Yvkj5294HsH84Z7v6LLuntcUnnqp65OZtQZvbomaUl3SXpAVV47hJ93asKzlsVV/6Fkt53\n9w/d/YKkLZKWVNBH13P3PZI+/sLiJZI2Zo83auR/no7L6a0ruPsJd9+fPf5U0qWZpSs9d4m+KlFF\n+K+VdHTU82Pqrim/XdIuM3vDzFZW3cwYerNp0yXppKTeKpsZQ8OZmzvpCzNLd825a2XG67Lxgd+X\nLXb3+ZK+J+lH2cvbruQj79m6abimqZmbO2WMmaX/ospz1+qM12WrIvzHJc0Z9fzr2bKu4O7Hs9+n\nJW1X980+fOrSJKnZ79MV9/MX3TRz81gzS6sLzl03zXhdRfj3Seo3s2+Y2WRJP5S0s4I+vsTMpmQf\nxMjMpkj6rrpv9uGdkpZnj5dL2lFhL5/TLTM3580srYrPXdfNeO3uHf+RdKdGPvH/QNI/VdFDTl/z\nJL2V/bxbdW+SntfIy8A/aeSzkQcl/bWk3ZKOSNolaWYX9facpHckva2RoM2uqLfFGnlJ/7akA9nP\nnVWfu0RflZw3vuEHBMUHfkBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgvp/UqBHBigpANMAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1bae6be9f60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  0.  0.  0.  0.  0.  0.  1.  0.  0.]\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=True)\n",
    "tmp = mnist.train.images[0]\n",
    "tmp = tmp.reshape((28,28))\n",
    "\n",
    "plt.imshow(tmp, cmap = cm.Greys)\n",
    "plt.show()\n",
    "\n",
    "\"\"\"\n",
    "A one-hot vector is a vector which is 0 in most dimensions, \n",
    "and 1 in a single dimension. In this case, the n-th digit will be represented as a \n",
    "vector which is 1 in the n-h dimension.\n",
    "\"\"\"\n",
    "\n",
    "print(mnist.train.labels[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.01\n",
    "training_epochs = 25\n",
    "batch_size = 100\n",
    "display_step = 1\n",
    "logs_path = '/tmp/tensorflow_logs/example'\n",
    "\n",
    "# tf Graph Input\n",
    "# mnist data image of shape 28*28=784\n",
    "\"\"\"\n",
    "When we assign None to our placeholder, it means the placeholder can be fed as \n",
    "many examples as you want to give it. \n",
    "In this case, our placeholder can be fed any multitude of 784-sized values.\n",
    "\"\"\"\n",
    "x = tf.placeholder(tf.float32, [None, 784], name='InputData')\n",
    "\n",
    "# 0-9 digits recognition => 10 classes\n",
    "# this is the placeholder where we will input our answers\n",
    "y = tf.placeholder(tf.float32, [None, 10], name='LabelData')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set model weights and bias\n",
    "\"\"\"\n",
    "We create these Variables by giving tf.Variable the initial value: \n",
    "in this case, we initialize both W and b as tensors full of zeros.\n",
    "Since we are going to learn W and b, it doesn't matter very much what they initially are.\n",
    "\"\"\"\n",
    "\n",
    "W = tf.Variable(tf.zeros([784, 10]), name='Weights')\n",
    "b = tf.Variable(tf.zeros([10]), name='Bias')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Construct model and encapsulating all ops into scopes, making\n",
    "# Tensorboard's Graph visualization more convenient\n",
    "with tf.name_scope('Model'):\n",
    "    # Model\n",
    "    # We make our prediction by multiplying each flattened digit by our weight and then adding the bias\n",
    "    pred = tf.nn.softmax(tf.matmul(x, W) + b) # Softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('Loss'):\n",
    "    # Minimize error using cross entropy\n",
    "    # The higher the cost, the higher the level of inaccuracy. \n",
    "    # It calculates accuracy by comparing the true values \n",
    "    # from y_train to the results of our prediction pred for each example. \n",
    "    # The goal is to minimize your loss.\n",
    "    cost = tf.reduce_mean(-tf.reduce_sum(y*tf.log(pred), reduction_indices=1))\n",
    "    # tf.reduce_sum adds the elements in the second dimension of y, due to the reduction_indices=[1] parameter\n",
    "    # tf.reduce_mean computes the mean over all the examples in the batch\n",
    "with tf.name_scope('SGD'):\n",
    "    # Gradient Descent to reduce the loss\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)\n",
    "with tf.name_scope('Accuracy'):\n",
    "    # Accuracy\n",
    "    # tf.argmax(y,1) is the label our model thinks is most likely for each input, \n",
    "    # while tf.argmax(y_,1) is the correct label\n",
    "    # we use tf.equal to check if our prediction matches the truth.\n",
    "    acc = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "    # That gives us a list of booleans. To determine what fraction are correct, \n",
    "    # we cast to floating point numbers and then take the mean\n",
    "    acc = tf.reduce_mean(tf.cast(acc, tf.float32))\n",
    "    # for example, [True, False, True, True] would become [1,0,1,1] which would become 0.75"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0001 cost= 1.184210822\n",
      "Epoch: 0002 cost= 0.665221844\n",
      "Epoch: 0003 cost= 0.552807781\n",
      "Epoch: 0004 cost= 0.498691760\n",
      "Epoch: 0005 cost= 0.465533523\n",
      "Epoch: 0006 cost= 0.442609970\n",
      "Epoch: 0007 cost= 0.425554885\n",
      "Epoch: 0008 cost= 0.412187044\n",
      "Epoch: 0009 cost= 0.401421105\n",
      "Epoch: 0010 cost= 0.392369530\n",
      "Epoch: 0011 cost= 0.384729818\n",
      "Epoch: 0012 cost= 0.378186758\n",
      "Epoch: 0013 cost= 0.372390979\n",
      "Epoch: 0014 cost= 0.367318740\n",
      "Epoch: 0015 cost= 0.362760928\n",
      "Epoch: 0016 cost= 0.358562535\n",
      "Epoch: 0017 cost= 0.354818366\n",
      "Epoch: 0018 cost= 0.351448950\n",
      "Epoch: 0019 cost= 0.348316518\n",
      "Epoch: 0020 cost= 0.345478419\n",
      "Epoch: 0021 cost= 0.342757122\n",
      "Epoch: 0022 cost= 0.340268253\n",
      "Epoch: 0023 cost= 0.337912548\n",
      "Epoch: 0024 cost= 0.335699880\n",
      "Epoch: 0025 cost= 0.333715808\n",
      "Optimization Finished!\n",
      "Accuracy: 0.9136\n"
     ]
    }
   ],
   "source": [
    "# Initializing the variables\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# Create a summary to monitor cost tensor\n",
    "tf.summary.scalar(\"loss\", cost)\n",
    "# Create a summary to monitor accuracy tensor\n",
    "tf.summary.scalar(\"accuracy\", acc)\n",
    "# Merge all summaries into a single op\n",
    "merged_summary_op = tf.summary.merge_all()\n",
    "\n",
    "# Launch the graph\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    # op to write logs to Tensorboard\n",
    "    summary_writer = tf.summary.FileWriter(logs_path, graph=tf.get_default_graph())\n",
    "\n",
    "    # Training cycle\n",
    "    for epoch in range(training_epochs):\n",
    "        avg_cost = 0.\n",
    "        total_batch = int(mnist.train.num_examples/batch_size)\n",
    "        # Loop over all batches\n",
    "        for i in range(total_batch):\n",
    "            batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "            # Run optimization op (backprop), cost op (to get loss value)\n",
    "            # and summary nodes\n",
    "            _, c, summary = sess.run([optimizer, cost, merged_summary_op],\n",
    "                                     feed_dict={x: batch_xs, y: batch_ys})\n",
    "            # Write logs at every iteration\n",
    "            summary_writer.add_summary(summary, epoch * total_batch + i)\n",
    "            # Compute average loss\n",
    "            avg_cost += c / total_batch\n",
    "        # Display logs per epoch step\n",
    "        if (epoch+1) % display_step == 0:\n",
    "            print(\"Epoch:\", '%04d' % (epoch+1), \"cost=\", \"{:.9f}\".format(avg_cost))\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "\n",
    "    # Test model\n",
    "    # Calculate accuracy\n",
    "    print(\"Accuracy:\", acc.eval({x: mnist.test.images, y: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"Resources/summary_tensorboard.png\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the summaries are also visualized in tensorboard\n",
    "\n",
    "from IPython.display import Image\n",
    "Image(url='Resources/summary_tensorboard.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.6092496441971565&quot;).pbtxt = 'node {\\n  name: &quot;InputData&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 784\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;LabelData&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: -1\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 784\\n          }\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Weights&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Weights/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;Weights&quot;\\n  input: &quot;zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@Weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Weights/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;Weights&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@Weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;zeros_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;Bias&quot;\\n  input: &quot;zeros_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@Bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;Bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@Bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Model/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;InputData&quot;\\n  input: &quot;Weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Model/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;Model/MatMul&quot;\\n  input: &quot;Bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Model/Softmax&quot;\\n  op: &quot;Softmax&quot;\\n  input: &quot;Model/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Loss/Log&quot;\\n  op: &quot;Log&quot;\\n  input: &quot;Model/Softmax&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Loss/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;LabelData&quot;\\n  input: &quot;Loss/Log&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Loss/Sum/reduction_indices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Loss/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;Loss/mul&quot;\\n  input: &quot;Loss/Sum/reduction_indices&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Loss/Neg&quot;\\n  op: &quot;Neg&quot;\\n  input: &quot;Loss/Sum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Loss/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;Loss/Neg&quot;\\n  input: &quot;Loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;SGD/gradients/Shape&quot;\\n  input: &quot;SGD/gradients/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;SGD/gradients/Fill&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;Loss/Neg&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Reshape&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;Loss/Neg&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Shape_1&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Shape_2&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Prod_1&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Prod&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Mean_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Tile&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Neg_grad/Neg&quot;\\n  op: &quot;Neg&quot;\\n  input: &quot;SGD/gradients/Loss/Mean_grad/truediv&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;Loss/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/Size&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 2\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;Loss/Sum/reduction_indices&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Size&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/mod&quot;\\n  op: &quot;FloorMod&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/add&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Size&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/range/start&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/range/delta&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/range&quot;\\n  op: &quot;Range&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/range/start&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Size&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/range/delta&quot;\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/Fill/value&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Shape_1&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Fill/value&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/DynamicStitch&quot;\\n  op: &quot;DynamicStitch&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/range&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/mod&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Shape&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Fill&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/DynamicStitch&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Shape&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;SGD/gradients/Loss/Neg_grad/Neg&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/DynamicStitch&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Sum_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Reshape&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/floordiv&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;LabelData&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;Loss/Log&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/Shape&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Tile&quot;\\n  input: &quot;Loss/Log&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/mul&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/Sum&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/mul_1&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;LabelData&quot;\\n  input: &quot;SGD/gradients/Loss/Sum_grad/Tile&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/mul_1&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/Sum_1&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^SGD/gradients/Loss/mul_grad/Reshape&quot;\\n  input: &quot;^SGD/gradients/Loss/mul_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/Reshape&quot;\\n  input: &quot;^SGD/gradients/Loss/mul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@SGD/gradients/Loss/mul_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/mul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/Reshape_1&quot;\\n  input: &quot;^SGD/gradients/Loss/mul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@SGD/gradients/Loss/mul_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Log_grad/Reciprocal&quot;\\n  op: &quot;Reciprocal&quot;\\n  input: &quot;Model/Softmax&quot;\\n  input: &quot;^SGD/gradients/Loss/mul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Loss/Log_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;SGD/gradients/Loss/mul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;SGD/gradients/Loss/Log_grad/Reciprocal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/Softmax_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;SGD/gradients/Loss/Log_grad/mul&quot;\\n  input: &quot;Model/Softmax&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/Softmax_grad/Sum/reduction_indices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/Softmax_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;SGD/gradients/Model/Softmax_grad/mul&quot;\\n  input: &quot;SGD/gradients/Model/Softmax_grad/Sum/reduction_indices&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/Softmax_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\377\\\\377\\\\377\\\\377\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/Softmax_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;SGD/gradients/Model/Softmax_grad/Sum&quot;\\n  input: &quot;SGD/gradients/Model/Softmax_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/Softmax_grad/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;SGD/gradients/Loss/Log_grad/mul&quot;\\n  input: &quot;SGD/gradients/Model/Softmax_grad/Reshape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/Softmax_grad/mul_1&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;SGD/gradients/Model/Softmax_grad/sub&quot;\\n  input: &quot;Model/Softmax&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;Model/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/Shape&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;SGD/gradients/Model/Softmax_grad/mul_1&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/Sum&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;SGD/gradients/Model/Softmax_grad/mul_1&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/Sum_1&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^SGD/gradients/Model/add_grad/Reshape&quot;\\n  input: &quot;^SGD/gradients/Model/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/Reshape&quot;\\n  input: &quot;^SGD/gradients/Model/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@SGD/gradients/Model/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/Reshape_1&quot;\\n  input: &quot;^SGD/gradients/Model/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@SGD/gradients/Model/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/tuple/control_dependency&quot;\\n  input: &quot;Weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;InputData&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^SGD/gradients/Model/MatMul_grad/MatMul&quot;\\n  input: &quot;^SGD/gradients/Model/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;SGD/gradients/Model/MatMul_grad/MatMul&quot;\\n  input: &quot;^SGD/gradients/Model/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@SGD/gradients/Model/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/gradients/Model/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;SGD/gradients/Model/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^SGD/gradients/Model/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@SGD/gradients/Model/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/GradientDescent/update_Weights/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;Weights&quot;\\n  input: &quot;SGD/GradientDescent/learning_rate&quot;\\n  input: &quot;SGD/gradients/Model/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@Weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/GradientDescent/update_Bias/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;Bias&quot;\\n  input: &quot;SGD/GradientDescent/learning_rate&quot;\\n  input: &quot;SGD/gradients/Model/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@Bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;SGD/GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^SGD/GradientDescent/update_Weights/ApplyGradientDescent&quot;\\n  input: &quot;^SGD/GradientDescent/update_Bias/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;Accuracy/ArgMax/dimension&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Accuracy/ArgMax&quot;\\n  op: &quot;ArgMax&quot;\\n  input: &quot;Model/Softmax&quot;\\n  input: &quot;Accuracy/ArgMax/dimension&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;output_type&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Accuracy/ArgMax_1/dimension&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Accuracy/ArgMax_1&quot;\\n  op: &quot;ArgMax&quot;\\n  input: &quot;LabelData&quot;\\n  input: &quot;Accuracy/ArgMax_1/dimension&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;output_type&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Accuracy/Equal&quot;\\n  op: &quot;Equal&quot;\\n  input: &quot;Accuracy/ArgMax&quot;\\n  input: &quot;Accuracy/ArgMax_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Accuracy/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;Accuracy/Equal&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Accuracy/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Accuracy/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;Accuracy/Cast&quot;\\n  input: &quot;Accuracy/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^Weights/Assign&quot;\\n  input: &quot;^Bias/Assign&quot;\\n}\\nnode {\\n  name: &quot;loss/tags&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;loss&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss&quot;\\n  op: &quot;ScalarSummary&quot;\\n  input: &quot;loss/tags&quot;\\n  input: &quot;Loss/Mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;accuracy/tags&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;accuracy&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;accuracy&quot;\\n  op: &quot;ScalarSummary&quot;\\n  input: &quot;accuracy/tags&quot;\\n  input: &quot;Accuracy/Mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Merge/MergeSummary&quot;\\n  op: &quot;MergeSummary&quot;\\n  input: &quot;loss&quot;\\n  input: &quot;accuracy&quot;\\n  attr {\\n    key: &quot;N&quot;\\n    value {\\n      i: 2\\n    }\\n  }\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.6092496441971565&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# to start your TensorBoard, you would first need to dump your data to disk\n",
    "writer = tf.summary.FileWriter(\"/tmp\")\n",
    "writer.add_graph(sess.graph)\n",
    "show_graph(tf.get_default_graph().as_graph_def())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# let's look at a high level abstraction library tflearn\n",
    "# pip install tflearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Step: 1000  | total loss: \u001b[1m\u001b[32m0.15393\u001b[0m\u001b[0m | time: 0.003s\n",
      "| SGD | epoch: 1000 | loss: 0.15393 - R2: 0.9743 -- iter: 17/17\n",
      "\n",
      "Regression result:\n",
      "Y = [ 0.25373101]*X + [ 0.78394109]\n"
     ]
    }
   ],
   "source": [
    "# simple linear regression example\n",
    "import tflearn\n",
    "from __future__ import absolute_import, division, print_function\n",
    "# Regression data\n",
    "X = [3.3,4.4,5.5,6.71,6.93,4.168,9.779,6.182,7.59,2.167,7.042,10.791,5.313,7.997,5.654,9.27,3.1]\n",
    "Y = [1.7,2.76,2.09,3.19,1.694,1.573,3.366,2.596,2.53,1.221,2.827,3.465,1.65,2.904,2.42,2.94,1.3]\n",
    "\n",
    "# Linear Regression graph\n",
    "input_ = tflearn.input_data(shape=[None])\n",
    "linear = tflearn.single_unit(input_)\n",
    "regression = tflearn.regression(linear, optimizer='sgd', loss='mean_square',\n",
    "                                metric='R2', learning_rate=0.01)\n",
    "m = tflearn.DNN(regression)\n",
    "m.fit(X, Y, n_epoch=1000, show_metric=True, snapshot_epoch=False)\n",
    "\n",
    "print(\"\\nRegression result:\")\n",
    "print(\"Y = \" + str(m.get_weights(linear.W)) +\n",
    "      \"*X + \" + str(m.get_weights(linear.b)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test prediction for x = 3.2, 3.3, 3.4:\n",
      "[ 1.59588027  1.62125349  1.64662647]\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nTest prediction for x = 3.2, 3.3, 3.4:\")\n",
    "print(m.predict([3.2, 3.3, 3.4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# another example with Titanic Dataset from Kaggle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hdf5 is not supported on this machine (please install/reinstall h5py for optimal experience)\n",
      "curses is not supported on this machine (please install/reinstall curses for an optimal experience)\n",
      "Downloading Titanic dataset...\n",
      "Succesfully downloaded titanic_dataset.csv 82865 bytes.\n",
      " class |  name |  sex |  age |  # of siblings on board |  # of parents/children on board |  ticket # |  fare |\n",
      "['1', 'Allison, Master. Hudson Trevor', 'male', '0.9167', '1', '2', '113781', '151.5500']\n",
      "['1', 'Allison, Miss. Helen Loraine', 'female', '2', '1', '2', '113781', '151.5500']\n",
      "['1', 'Allison, Mr. Hudson Joshua Creighton', 'male', '30', '1', '2', '113781', '151.5500']\n",
      "['1', 'Allison, Mrs. Hudson J C (Bessie Waldo Daniels)', 'female', '25', '1', '2', '113781', '151.5500']\n",
      "['1', 'Anderson, Mr. Harry', 'male', '48', '0', '0', '19952', '26.5500']\n",
      "['1', 'Andrews, Miss. Kornelia Theodosia', 'female', '63', '1', '0', '13502', '77.9583']\n",
      "['1', 'Andrews, Mr. Thomas Jr', 'male', '39', '0', '0', '112050', '0.0000']\n",
      "['1', 'Appleton, Mrs. Edward Dale (Charlotte Lamson)', 'female', '53', '2', '0', '11769', '51.4792']\n",
      "['1', 'Artagaveytia, Mr. Ramon', 'male', '71', '0', '0', 'PC 17609', '49.5042']\n"
     ]
    }
   ],
   "source": [
    "# http://tflearn.org/tutorials/quickstart.html\n",
    "import numpy as np\n",
    "import tflearn\n",
    "# make sure you have pandas=0.19.2 in your virtual environment for this example to work\n",
    "# conda install pandas=0.19.2\n",
    "\n",
    "# Download the Titanic dataset\n",
    "from tflearn.datasets import titanic\n",
    "titanic.download_dataset('titanic_dataset.csv')\n",
    "\n",
    "# Load CSV file, indicate that the first column represents labels\n",
    "from tflearn.data_utils import load_csv\n",
    "data, labels = load_csv('titanic_dataset.csv', target_column=0,\n",
    "                        categorical_labels=True, n_classes=2)\n",
    "\n",
    "# let's look at the data\n",
    "\n",
    "# print column names\n",
    "print(\" class |\", \" name |\", \" sex |\", \" age |\", \" # of siblings on board |\", \" # of parents/children on board |\", \" ticket # |\", \" fare |\")\n",
    "\n",
    "# print out some data samples\n",
    "for element in data[1:10]:\n",
    "    print(element)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocessing function\n",
    "def preprocess(data, columns_to_ignore):\n",
    "    # Sort by descending id and delete columns\n",
    "    for id in sorted(columns_to_ignore, reverse=True):\n",
    "        [r.pop(id) for r in data]\n",
    "    for i in range(len(data)):\n",
    "        # Converting 'sex' field to float (id is 1 after removing labels column)\n",
    "        data[i][1] = 1. if data[i][1] == 'female' else 0.\n",
    "    return np.array(data, dtype=np.float32)\n",
    "\n",
    "# Ignore 'name' and 'ticket' columns (id 1 & 6 of data array)\n",
    "to_ignore=[1, 6]\n",
    "\n",
    "# Preprocess data\n",
    "data = preprocess(data, to_ignore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Step: 819  | total loss: \u001b[1m\u001b[32m0.54480\u001b[0m\u001b[0m | time: 0.299s\n",
      "| Adam | epoch: 010 | loss: 0.54480 - acc: 0.7566 -- iter: 1296/1309\n",
      "Training Step: 820  | total loss: \u001b[1m\u001b[32m0.51864\u001b[0m\u001b[0m | time: 0.302s\n",
      "| Adam | epoch: 010 | loss: 0.51864 - acc: 0.7684 -- iter: 1309/1309\n",
      "--\n"
     ]
    }
   ],
   "source": [
    "# Build neural network\n",
    "net = tflearn.input_data(shape=[None, 6])\n",
    "net = tflearn.fully_connected(net, 32)\n",
    "net = tflearn.fully_connected(net, 32)\n",
    "net = tflearn.fully_connected(net, 2, activation='softmax')\n",
    "net = tflearn.regression(net)\n",
    "\n",
    "# Define model\n",
    "model = tflearn.DNN(net)\n",
    "# Start training (apply gradient descent algorithm)\n",
    "model.fit(data, labels, n_epoch=10, batch_size=16, show_metric=True)\n",
    "\n",
    "# if this doesn't work, restart your notebook kernel!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DiCaprio Surviving Rate: 0.136774\n",
      "Winslet Surviving Rate: 0.884572\n"
     ]
    }
   ],
   "source": [
    "# Let's create some data for DiCaprio and Winslet\n",
    "dicaprio = [3, 'Jack Dawson', 'male', 19, 0, 0, 'N/A', 5.0000]\n",
    "winslet = [1, 'Rose DeWitt Bukater', 'female', 17, 1, 2, 'N/A', 100.0000]\n",
    "# Preprocess data\n",
    "dicaprio, winslet = preprocess([dicaprio, winslet], to_ignore)\n",
    "# Predict surviving chances (class 1 results)\n",
    "pred = model.predict([dicaprio, winslet])\n",
    "print(\"DiCaprio Surviving Rate:\", pred[0][1])\n",
    "print(\"Winslet Surviving Rate:\", pred[1][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}