01_Train_Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Model with CPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.client import timeline\n",
    "import pylab\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reset TensorFlow Graph\n",
    "Useful in Jupyter Notebooks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create TensorFlow Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "print(sess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Model Training and Test/Validation Data \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_samples = 100000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "x_train = np.random.rand(num_samples).astype(np.float32)\n",
    "print(x_train)\n",
    "\n",
    "noise = np.random.normal(scale=0.01, size=len(x_train))\n",
    "\n",
    "y_train = x_train * 0.1 + 0.3 + noise\n",
    "print(y_train)\n",
    "\n",
    "pylab.plot(x_train, y_train, '.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pylab\n",
    "\n",
    "x_test = np.random.rand(len(x_train)).astype(np.float32)\n",
    "print(x_test)\n",
    "\n",
    "noise = np.random.normal(scale=.01, size=len(x_train))\n",
    "\n",
    "y_test = x_test * 0.1 + 0.3 + noise\n",
    "print(y_test)\n",
    "\n",
    "pylab.plot(x_test, y_test, '.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(\"/cpu:0\"):\n",
    "    W = tf.get_variable(shape=[], name='weights')\n",
    "    print(W)\n",
    "\n",
    "    b = tf.get_variable(shape=[], name='bias')\n",
    "    print(b)\n",
    "\n",
    "    x_observed = tf.placeholder(shape=[None], \n",
    "                                dtype=tf.float32, \n",
    "                                name='x_observed')\n",
    "    print(x_observed)\n",
    "\n",
    "    y_pred = W * x_observed + b\n",
    "    print(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.025\n",
    "\n",
    "with tf.device(\"/cpu:0\"):\n",
    "    y_observed = tf.placeholder(shape=[None], dtype=tf.float32, name='y_observed')\n",
    "    print(y_observed)\n",
    "\n",
    "    loss_op = tf.reduce_mean(tf.square(y_pred - y_observed))\n",
    "    optimizer_op = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    \n",
    "    train_op = optimizer_op.minimize(loss_op)  \n",
    "\n",
    "    print(\"Loss Scalar: \", loss_op)\n",
    "    print(\"Optimizer Op: \", optimizer_op)\n",
    "    print(\"Train Op: \", train_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Randomly Initialize Variables (Weights and Bias)\n",
    "The goal is to learn more accurate Weights and Bias during training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(\"/cpu:0\"):\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    print(init_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.run(init_op)\n",
    "print(\"Initial random W: %f\" % sess.run(W))\n",
    "print(\"Initial random b: %f\" % sess.run(b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## View Accuracy of Pre-Training, Initial Random Variables\n",
    "We want this to be close to 0, but it's relatively far away.  This is why we train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(x, y):\n",
    "    return sess.run(loss_op, feed_dict={x_observed: x, y_observed: y})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test(x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Loss Summary Operations for Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_summary_scalar_op = tf.summary.scalar('loss', loss_op)\n",
    "loss_summary_merge_all_op = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_summary_writer = tf.summary.FileWriter('./linear_model/logs/cpu/train/', \n",
    "                                            graph=tf.get_default_graph())\n",
    "\n",
    "test_summary_writer = tf.summary.FileWriter('./linear_model/logs/cpu/test/',\n",
    "                                            graph=tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "from tensorflow.python.client import timeline\n",
    "\n",
    "with tf.device(\"/cpu:0\"):\n",
    "    run_metadata = tf.RunMetadata()\n",
    "    max_steps = 401\n",
    "    for step in range(max_steps):\n",
    "        if (step < max_steps - 1):\n",
    "            test_summary_log, _ = sess.run([loss_summary_merge_all_op, loss_op], feed_dict={x_observed: x_test, y_observed: y_test})\n",
    "            train_summary_log, _ = sess.run([loss_summary_merge_all_op, train_op], feed_dict={x_observed: x_train, y_observed: y_train})\n",
    "        else:  \n",
    "            test_summary_log, _ = sess.run([loss_summary_merge_all_op, loss_op], feed_dict={x_observed: x_test, y_observed: y_test})\n",
    "            train_summary_log, _ = sess.run([loss_summary_merge_all_op, train_op], feed_dict={x_observed: x_train, y_observed: y_train}, \n",
    "                                            options=tf.RunOptions(trace_level=tf.RunOptions.SOFTWARE_TRACE), \n",
    "                                            run_metadata=run_metadata)\n",
    "\n",
    "            trace = timeline.Timeline(step_stats=run_metadata.step_stats)    \n",
    "            with open('timeline-cpu.json', 'w') as trace_file:\n",
    "                trace_file.write(trace.generate_chrome_trace_format(show_memory=True))\n",
    "\n",
    "        if step % 10 == 0:\n",
    "            print(step, sess.run([W, b]))\n",
    "            train_summary_writer.add_summary(train_summary_log, step)\n",
    "            train_summary_writer.flush()\n",
    "            test_summary_writer.add_summary(test_summary_log, step)\n",
    "            test_summary_writer.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pylab.plot(x_train, y_train, '.', label=\"target\")\n",
    "pylab.plot(x_train, sess.run(y_pred, \n",
    "                             feed_dict={x_observed: x_train, \n",
    "                                        y_observed: y_train}), \n",
    "           \".\", \n",
    "           label=\"predicted\")\n",
    "pylab.legend()\n",
    "pylab.ylim(0, 1.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## View Loss Summaries in Tensorboard\n",
    "Navigate to the **`Scalars`** and **`Graphs`** tab in TensorBoard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Graph For Optimization\n",
    "We will use this later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "checkpoint_base_path = './linear_model/cpu/checkpoint'\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "graph_model_path = '%s/graph.pb' % checkpoint_base_path\n",
    "print(graph_model_path)\n",
    "\n",
    "os.makedirs(checkpoint_base_path, exist_ok=True)\n",
    "\n",
    "tf.train.write_graph(sess.graph_def, \n",
    "                     '.', \n",
    "                     graph_model_path,\n",
    "                     as_text=False) \n",
    "\n",
    "\n",
    "checkpoint_model_path = '%s/model.ckpt' % checkpoint_base_path\n",
    "saver.save(sess, \n",
    "           save_path=checkpoint_model_path)\n",
    "\n",
    "print(checkpoint_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.listdir(checkpoint_base_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "import re\n",
    "from google.protobuf import text_format\n",
    "from tensorflow.core.framework import graph_pb2\n",
    "\n",
    "def convert_graph_to_dot(input_graph, output_dot, is_input_graph_binary):\n",
    "    graph = graph_pb2.GraphDef()\n",
    "    with open(input_graph, \"rb\") as fh:\n",
    "        if is_input_graph_binary:\n",
    "            graph.ParseFromString(fh.read())\n",
    "        else:\n",
    "            text_format.Merge(fh.read(), graph)\n",
    "    with open(output_dot, \"wt\") as fh:\n",
    "        print(\"digraph graphname {\", file=fh)\n",
    "        for node in graph.node:\n",
    "            output_name = node.name\n",
    "            print(\"  \\\"\" + output_name + \"\\\" [label=\\\"\" + node.op + \"\\\"];\", file=fh)\n",
    "            for input_full_name in node.input:\n",
    "                parts = input_full_name.split(\":\")\n",
    "                input_name = re.sub(r\"^\\^\", \"\", parts[0])\n",
    "                print(\"  \\\"\" + input_name + \"\\\" -> \\\"\" + output_name + \"\\\";\", file=fh)\n",
    "        print(\"}\", file=fh)\n",
    "        print(\"Created dot file '%s' for graph '%s'.\" % (output_dot, input_graph))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dot_path='./graph_cpu.dot'\n",
    "convert_graph_to_dot(input_graph=graph_model_path, output_dot=output_dot_path, is_input_graph_binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash -s \"$output_dot_path\"\n",
    "\n",
    "dot -T png $1 \\\n",
    "    -o ./graph_cpu.png > /tmp/a.out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import Image\n",
    "\n",
    "Image('./graph_cpu.png', width=1024, height=768)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}