notebooks/01_model_review.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1001\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "        toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1001\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
       "        if (callback != null)\n",
       "          callback();\n",
       "      });\n",
       "    } finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.debug(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(css_urls, js_urls, callback) {\n",
       "    if (css_urls == null) css_urls = [];\n",
       "    if (js_urls == null) js_urls = [];\n",
       "\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
       "\n",
       "    function on_load() {\n",
       "      root._bokeh_is_loading--;\n",
       "      if (root._bokeh_is_loading === 0) {\n",
       "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
       "        run_callbacks()\n",
       "      }\n",
       "    }\n",
       "\n",
       "    function on_error() {\n",
       "      console.error(\"failed to load \" + url);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < css_urls.length; i++) {\n",
       "      var url = css_urls[i];\n",
       "      const element = document.createElement(\"link\");\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.rel = \"stylesheet\";\n",
       "      element.type = \"text/css\";\n",
       "      element.href = url;\n",
       "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
       "      document.body.appendChild(element);\n",
       "    }\n",
       "\n",
       "    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\": \"qkRvDQVAIfzsJo40iRBbxt6sttt0hv4lh74DG7OK4MCHv4C5oohXYoHUM5W11uqS\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\": \"Sb7Mr06a9TNlet/GEBeKaf5xH3eb6AlCzwjtU82wNPyDrnfoiVl26qnvlKjmcAd+\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\": \"HaJ15vgfmcfRtB4c4YBOI4f1MUujukqInOWVqZJZZGK7Q+ivud0OKGSTn/Vm2iso\"};\n",
       "\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var element = document.createElement('script');\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.async = false;\n",
       "      element.src = url;\n",
       "      if (url in hashes) {\n",
       "        element.crossOrigin = \"anonymous\";\n",
       "        element.integrity = \"sha384-\" + hashes[url];\n",
       "      }\n",
       "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.head.appendChild(element);\n",
       "    }\n",
       "  };\n",
       "\n",
       "  function inject_raw_css(css) {\n",
       "    const element = document.createElement(\"style\");\n",
       "    element.appendChild(document.createTextNode(css));\n",
       "    document.body.appendChild(element);\n",
       "  }\n",
       "\n",
       "  \n",
       "  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\"];\n",
       "  var css_urls = [];\n",
       "  \n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    function(Bokeh) {\n",
       "    \n",
       "    \n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if (root.Bokeh !== undefined || force === true) {\n",
       "      \n",
       "    for (var i = 0; i < inline_js.length; i++) {\n",
       "      inline_js[i].call(root, root.Bokeh);\n",
       "    }\n",
       "    if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(css_urls, js_urls, function() {\n",
       "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1001\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    const hashes = {\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\": \"qkRvDQVAIfzsJo40iRBbxt6sttt0hv4lh74DG7OK4MCHv4C5oohXYoHUM5W11uqS\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\": \"Sb7Mr06a9TNlet/GEBeKaf5xH3eb6AlCzwjtU82wNPyDrnfoiVl26qnvlKjmcAd+\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\": \"HaJ15vgfmcfRtB4c4YBOI4f1MUujukqInOWVqZJZZGK7Q+ivud0OKGSTn/Vm2iso\"};\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      if (url in hashes) {\n        element.crossOrigin = \"anonymous\";\n        element.integrity = \"sha384-\" + hashes[url];\n      }\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.2.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.2.1.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import gzip\n",
    "import io\n",
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pandas_profiling\n",
    "import seaborn as sns\n",
    "import statsmodels.api as sm\n",
    "import xgboost as xgb\n",
    "from bokeh.io import output_notebook\n",
    "from bokeh.layouts import column\n",
    "from bokeh.models import Band, ColumnDataSource, HoverTool, NumeralTickFormatter, Select\n",
    "from bokeh.plotting import figure, gridplot, show\n",
    "from fancyimpute import KNN, NuclearNormMinimization, SoftImpute, BiScaler\n",
    "from s3fs import S3FileSystem\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import GridSearchCV, learning_curve, ShuffleSplit, train_test_split\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "\n",
    "RANDOM_STATE = 42\n",
    "output_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Model Review: Taking a model from exploration to production\n",
    "  \n",
    "<span style=\"display: inline-block; height:200px; width:720px\"></span>\n",
    "  \n",
    "  \n",
    "[https://github.com/aterrel/model-review-talk](https://github.com/aterrel/model-review-talk)\n",
    "    \n",
    "Presented by:  \n",
    "[Andy R. Terrel, PhD](https://www.linkedin.com/in/aterrel/)  | Chief Data Scientist, [REX Inc.](https://rexhomes.com) | President, [NumFOCUS](https://numfocus.org)  \n",
    "\n",
    "Contributions by:  \n",
    "[Andy Maloney](https://linkedin.com/in/andy-maloney-a43a34195) | [John Hanley](https://linkedin.com/in/jhanley714) | REX Data Team"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Model is Done! NOW WHAT?\n",
    "\n",
    "<span style=\"display: inline-block; height:25px; width:720px\"></span>\n",
    "\n",
    "<div style=\"display: flex;\">\n",
    "<div style=\"height:250px; width:15%\"></div>\n",
    "<div>\n",
    "<img src=\"../images/austin-neill-emH2e5SBifE-unsplash.jpg\" width=\"1250\">\n",
    "    </div>\n",
    "</div>\n",
    "\n",
    "\n",
    "<span style=\"font-size: small\">Photo by <a href=\"https://unsplash.com/@arstyy?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText\">Austin Neill</a> on <a href=\"https://unsplash.com/s/photos/ship?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText\">Unsplash</a></span>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<img src=\"../images/sculley-et-al_hidden-tech-debt-ml.png\" width=1500>\n",
    "\n",
    "<span style=\"font-size: small\">Figure from Sculley, D & Holt, Gary & Golovin, Daniel & Davydov, Eugene & Phillips, Todd & Ebner, Dietmar & Chaudhary, Vinay & Young, Michael & Dennison, Dan. (2015). Hidden Technical Debt in Machine Learning Systems. NIPS. 2494-2502. <a href=\"https://papers.nips.cc/paper/5656-hidden-technical-debt-in-machine-learning-systems.pdf\">https://papers.nips.cc/paper/5656-hidden-technical-debt-in-machine-learning-systems.pdf</a></span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Charting our journey:\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "- Understanding your models deployment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "- Tracking data into and out of your model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "- Detecting problems with your model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "- Pulling it together in a checklist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Understanding your models deployment\n",
    "\n",
    "<span style=\"display: inline-block; height:25px; width:720px\"></span>\n",
    "\n",
    "<div style=\"display: flex;\">\n",
    "<div style=\"height:250px; width:25%\"></div>\n",
    "<div>\n",
    "<img src=\"../images/guillaume-bolduc-uBe2mknURG4-unsplash.jpg\" width=\"950\">\n",
    "    </div>\n",
    "</div>\n",
    "\n",
    "<span style=\"font-size: small\">Photo by <a href=\"https://unsplash.com/@guibolduc?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText\">Guillaume Bolduc</a> on <a href=\"https://unsplash.com/s/photos/ship?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText\">Unsplash</a></span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Data collection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### **`DATA SOURCES`**\n",
    "\n",
    "  List all data sources, _e.g._. This should help you understand what data can\n",
    "  be used for the analysis/model, and how to ultimately do the ETL for the\n",
    "  model.\n",
    "\n",
    "  * Application database\n",
    "  * Customer Relations Manager records\n",
    "  * Web analytic events\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### PROVENANCE\n",
    "\n",
    "  Basically you need to show _and store_ the following Entity, Activity, and Agent information about the notebook/models.\n",
    "  \n",
    "  <img src=\"../images/provbook-example.png\" width=720>\n",
    "  \n",
    "  <span style=\"font-size: small\">Dong Huynh,\n",
    "    http://trungdong.github.io/prov-python-short-tutorial.html</span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "  #### PROVENANCE\n",
    "\n",
    "  * Entity\n",
    "\n",
    "    * An entity is a physical, digital, conceptual, or other kind of thing\n",
    "      with some fixed aspects; entities may be real or imaginary.\n",
    "\n",
    "  * Activity\n",
    "\n",
    "    * An activity is something that occurs over a period of time and acts upon\n",
    "      or with entities; it may include consuming, processing, transforming,\n",
    "      modifying, relocating, using, or generating entities.\n",
    "\n",
    "  * Agent\n",
    "\n",
    "    * An agent is something that bears some form of responsibility for an\n",
    "      activity taking place, for the existence of an entity, or for another\n",
    "      agent's activity."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Testing contracts into and out of your model\n",
    "\n",
    "- During model building, you discovered many things about features:\n",
    "  + range of feature\n",
    "  + distribution of feature\n",
    "  + sensitivity to feature variance\n",
    "- These learnings can be codified into preprocessing stages of your model\n",
    "- Additionally, you can monitor the predictions of your model in the same fashion, detecting when your predictions start to be biased\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Prediction Environment\n",
    "\n",
    "\n",
    "- How do you get called?\n",
    "- What is the SLA your code needs to adhere to?\n",
    "- What are the systems monitoring the model?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### How do you get called\n",
    "\n",
    "<span style=\"display: inline-block; height:25px; width:720px\"></span>\n",
    "\n",
    "<div style=\"display: flex;\">\n",
    "<div style=\"height:250px; width:25%\"></div>\n",
    "<div>\n",
    "<img src=\"../images/richards-event-architecture.png\" width=\"950\">\n",
    "    </div>\n",
    "</div>\n",
    "\n",
    "<span style=\"font-size: small\">Figure from Mark Richards, __Software Architecture Patterns__</span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### The other places you get called\n",
    "\n",
    "\n",
    "<span style=\"display: inline-block; height:25px; width:720px\"></span>\n",
    "\n",
    "<div style=\"display: flex;\">\n",
    "<div style=\"height:250px; width:25%\"></div>\n",
    "<div>\n",
    "<img src=\"../images/mapr-lambda-architecture.png\" width=\"1250\">\n",
    "    </div>\n",
    "</div>\n",
    "\n",
    "\n",
    "<span style=\"font-size: small\">Figure by MapR</span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Who is monitoring you?\n",
    "\n",
    "<span style=\"display: inline-block; height:25px; width:720px\"></span>\n",
    "\n",
    "<div style=\"display: flex;\">\n",
    "<div style=\"height:250px; width:25%\"></div>\n",
    "<div>\n",
    "<img src=\"../images/terrel-breakdown-of-on-node-monitors.png\" width=\"750\">\n",
    "    </div>\n",
    "</div>\n",
    "\n",
    "<span style=\"font-size: small\">Figure by Andy Terrel</span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## How did you get there ?!\n",
    "\n",
    "<span style=\"display: inline-block; height:25px; width:720px\"></span>\n",
    "\n",
    "<div style=\"display: flex;\">\n",
    "<div style=\"height:250px; width:25%\"></div>\n",
    "<div>\n",
    "<img src=\"../images/noaa-3duT-54VuK8-unsplash.jpg\" width=\"950\">\n",
    "    </div>\n",
    "</div>\n",
    "\n",
    "<span style=\"font-size: small\">Photo by <a href=\"https://unsplash.com/@noaa?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText\">NOAA</a> on <a href=\"https://unsplash.com/s/photos/ship?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText\">Unsplash</a></span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Data provenance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "class Extract:\n",
    "    \"\"\"This is an example of an Agent. This object is going to be a recipe\n",
    "    for the extraction of data. It will not do anything by itself, since the\n",
    "    extraction is done through the _extract method that must be written by a\n",
    "    class that inherits this object.\n",
    "    \"\"\"\n",
    "\n",
    "    def _extract(self, *args, **kwargs):\n",
    "        raise NotImplementedError('To be implemented by the inheriting class.')\n",
    "\n",
    "    def extract(self, *args, **kwargs):\n",
    "        extracted_data = self._extract(*args, **kwargs)\n",
    "        return extracted_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "class Transform:\n",
    "    \"\"\"This is another Agent that we will use to transform any extracted data.\"\"\"\n",
    "\n",
    "    def _transform(self, *args, **kwargs):\n",
    "        raise NotImplementedError('To be implemented by the inheriting class.')\n",
    "\n",
    "    def transform(self, *args, **kwargs):\n",
    "        transformed_data = self._transform(*args, **kwargs)\n",
    "        return transformed_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "class Load:\n",
    "    \"\"\"Just as in the Extract and Transform objects, we create a Load object\n",
    "    that is a yet another Agent. We will use this data loading object to build\n",
    "    another data set object that will be used to create entities for our\n",
    "    model(s).\n",
    "    \"\"\"\n",
    "\n",
    "    def _load(self, *args, **kwargs):\n",
    "        raise NotImplementedError('To be implemented by the inheriting class.')\n",
    "\n",
    "    def load(self, *args, **kwargs):\n",
    "        loaded_data = self._load(*args, **kwargs)\n",
    "        return loaded_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'S3FileSystem' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-3b60933b5d24>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mCSVDataSet\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \"\"\"This example will show how to persist a data set locally and on S3. This\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mis\u001b[0m \u001b[0man\u001b[0m \u001b[0mAgent\u001b[0m \u001b[0mthat\u001b[0m \u001b[0morchestrates\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mloading\u001b[0m \u001b[0mof\u001b[0m \u001b[0mdata\u001b[0m \u001b[0mset\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mthe\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mcreation\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mdata\u001b[0m \u001b[0mset\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-5-3b60933b5d24>\u001b[0m in \u001b[0;36mCSVDataSet\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0mdata_set_path_local\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0mdata_set_path_remote\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m     \u001b[0ms3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mS3FileSystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'S3FileSystem' is not defined"
     ]
    }
   ],
   "source": [
    "class CSVDataSet:\n",
    "    \"\"\"This example will show how to persist a data set locally and on S3. This\n",
    "    is an Agent that orchestrates the loading of data set and the\n",
    "    creation of the data set object.\n",
    "\n",
    "    Again, this object will not do anything useful since it requires bucket\n",
    "    names and file names to be created, as well as the _load method.\n",
    "    \"\"\"\n",
    "\n",
    "    name = None\n",
    "    data_set_name = None\n",
    "    data_set_file_extension = 'csv'\n",
    "    file_compression = 'gz'\n",
    "    data_set_file_name = None\n",
    "    bucket_local = None\n",
    "    bucket_remote = None\n",
    "    bucket_prefix = None\n",
    "    data_set_path_local = None\n",
    "    data_set_path_remote = None\n",
    "    s3 = S3FileSystem()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "    def cache_df(self, df, where):\n",
    "        print(f'Caching data set {where}.')\n",
    "        data_set_buffer = io.StringIO()\n",
    "        df.to_csv(data_set_buffer, index=False)\n",
    "        data_set_buffer.seek(0)\n",
    "        gzipped_data_set_buffer = io.BytesIO()\n",
    "        with gzip.GzipFile(mode='w', fileobj=gzipped_data_set_buffer) as file_:\n",
    "            file_.write(bytes(data_set_buffer.getvalue(), 'utf-8'))\n",
    "\n",
    "        if where == 'locally':\n",
    "            with open(self.data_set_path_local, 'wb') as file_:\n",
    "                file_.write(gzipped_data_set_buffer.getvalue())\n",
    "\n",
    "        elif where == 'remotely':\n",
    "            with self.s3.open(self.data_set_path_remote, 'wb') as file_:\n",
    "                file_.write(gzipped_data_set_buffer.getvalue())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "    def _load_data_set(self, *args, **kwargs):\n",
    "        if 'overwrite_cache' in kwargs:\n",
    "            overwrite_cache = kwargs.get('overwrite_cache', False)\n",
    "        else:\n",
    "            overwrite_cache = False\n",
    "            \n",
    "        ...\n",
    "\n",
    "        if data_set_exists_locally:\n",
    "            print('Loading the data set from the local cache.')\n",
    "            self.df = pd.read_csv(self.data_set_path_local, low_memory=False)\n",
    "            if not data_set_exists_remotely:\n",
    "                self.cache_df(df=self.df, where='remotely')\n",
    "                \n",
    "        ...\n",
    "\n",
    "    def load_data_set(self, *args, **kwargs):\n",
    "        self._load_data_set(*args, **kwargs)\n",
    "\n",
    "    def is_data_set_cached(self):\n",
    "        return not self.df.empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "class IrisDataExtractor(Extract):\n",
    "    \"\"\"This is an example of an Activity. It will require the inheritance of\n",
    "    the agent called Extract. Here we will write the actual extraction of the\n",
    "    data. This object could perform the generation of an Entity, if the\n",
    "    extraction is a time-consuming process.\n",
    "    \"\"\"\n",
    "\n",
    "    def _extract(self):\n",
    "        iris_data = load_iris()\n",
    "        return iris_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "class IrisDataTransformer(Transform):\n",
    "    \"\"\"As was done with the data extraction, here we will write the data\n",
    "    transformation. This is another example of an Activity. It should be\n",
    "    noted that this is not a generic transformation class. It is dependent\n",
    "    on the extracted Iris data set. If you make this class a generic data\n",
    "    transformation, then you need to make sure that any assumptions about the\n",
    "    model are described further in the process. Making this \"generic\" could be\n",
    "    useful if you need to make many models with different assumptions in them\n",
    "    that are all based on the same data.\n",
    "\n",
    "    If you do not need to make this generic, then it is a good idea to place\n",
    "    all assumptions about the model and data in this object.\n",
    "    \"\"\"\n",
    "\n",
    "    def _transform(self, data):\n",
    "        df = pd.DataFrame(\n",
    "            data=data['data'],\n",
    "            columns=data['feature_names'],\n",
    "        )\n",
    "        name_dict = {i: name for i, name in enumerate(data['target_names'])}\n",
    "        df['iris_name'] = data['target']\n",
    "        df['iris_name'] = df['iris_name'].apply(lambda index: name_dict[index])\n",
    "        return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-54809401a317>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mIrisDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLoad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \"\"\"This specific example of an Activity also sets the dataframe attribute\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mas\u001b[0m \u001b[0man\u001b[0m \u001b[0mEntity\u001b[0m \u001b[0mto\u001b[0m \u001b[0mthis\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mThis\u001b[0m \u001b[0mmay\u001b[0m \u001b[0mseem\u001b[0m \u001b[0munusual\u001b[0m \u001b[0mright\u001b[0m \u001b[0mnow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbut\u001b[0m \u001b[0mit\u001b[0m \u001b[0mwill\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mmake\u001b[0m \u001b[0msense\u001b[0m \u001b[0mwhen\u001b[0m \u001b[0mused\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconjunction\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mDataSet\u001b[0m \u001b[0mobject\u001b[0m \u001b[0mbelow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \"\"\"\n",
      "\u001b[0;32m<ipython-input-8-54809401a317>\u001b[0m in \u001b[0;36mIrisDataLoader\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m     \"\"\"\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_load\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "class IrisDataLoader(Load):\n",
    "    \"\"\"This specific example of an Activity also sets the dataframe attribute\n",
    "    as an Entity to this object. This may seem unusual right now, but it will\n",
    "    make sense when used in conjunction with the DataSet object below.\n",
    "    \"\"\"\n",
    "\n",
    "    df = pd.DataFrame()\n",
    "\n",
    "    def _load(self):\n",
    "        extractor = IrisDataExtractor()\n",
    "        extracted_data = extractor.extract()\n",
    "        transformer = IrisDataTransformer()\n",
    "        transformed_data = transformer.transform(extracted_data)\n",
    "        self.df = transformed_data.copy()\n",
    "        return self.df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'CSVDataSet' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-52fe6e88ab96>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mIrisDataSet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCSVDataSet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mIrisDataLoader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mbucket_local\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'../data/'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mbucket_remote\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'org-rex-data'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mbucket_prefix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'research/rex-analysis/amaloney'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'CSVDataSet' is not defined"
     ]
    }
   ],
   "source": [
    "class IrisDataSet(CSVDataSet, IrisDataLoader):\n",
    "\n",
    "    bucket_local = '../data/'\n",
    "    bucket_remote = 'org-rex-data'\n",
    "    bucket_prefix = 'research/rex-analysis/amaloney'\n",
    "\n",
    "    def __init__(self):\n",
    "        self.generate_names_with_cache_paths()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Write down your Meta model\n",
    "\n",
    "**objective** – a sentence or two on what your model or analysis aims at.\n",
    "\n",
    "**KPIs* – List all key performance indicators. For example:\n",
    "\n",
    "- Model predicts X.\n",
    "- Further actions that need to be taken to acquire data for the model.\n",
    "- Out of sample predictions show that...\n",
    "- Y is an outlier when these assumptions are made...\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**inputs**\n",
    "- Tell us what the model reads, e.g. postgres_db_uri , or an S3 location like https://s3.console.aws.amazon.com/s3/buckets/bar/baz\n",
    "- What is the provenance? Directly querying a System of Record? Or a subsequent journey? Can we reproduce it?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**outputs** \n",
    "- Where will inferences go? Stdout, file, S3, a table? \n",
    "- Should we be worried about overwriting some frozen output?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**assumptions**\n",
    "\n",
    "- Are assumptions explicitly written down in a ReadMe? \n",
    "- The most common violations \n",
    "  - assuming independence over observations that aren't independent, or \n",
    "  - implicitly assuming some statistical distribution such as normality.\n",
    "\n",
    "- Do assumptions actually hold in practice, in the observed data? \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**benchmark** \n",
    "\n",
    "- What competing model are you comparing performance to? \n",
    "- Is it the best available?\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**bias-variance tradeoff**\n",
    "- What does the learning curve say about overfitting?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**transformation** \n",
    "- How is data transformed during ETL? \n",
    "- Filtering? \n",
    "- Missing attributes? \n",
    "- Imputation?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Showing your Data Prep\n",
    "\n",
    "\n",
    "<span style=\"display: inline-block; height:700px; width:2000px\"></span>\n",
    "<span style=\"font-size: small\">cue live demo</span>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### The Review Checklist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Summary\n",
    "\n",
    "Author will create a request file, and Reviewer will create a result file, which are permanently added to the repo.\n",
    "\n",
    "#### Outcome\n",
    "\n",
    "At end of review, we’ll have learned whether Reviewer believes the model (believes it is useful), and finds it easy to use."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Prerequisites\n",
    "\n",
    "- You have a model checked into source control. Good! \n",
    "- You have an **Objective**, in the form of a README.md or similar file. It, too, is checked into the repo.\n",
    "- You have observations. Since they are “big”, too big to conveniently feed to the elephant named Git, they can be found in S3, or perhaps in an RDBMS table. They are frozen, they shall not change in the next week or two. Consider putting them in an S3 object or DB table that has an iso8601 date as part of the name, e.g. foo-2020-04-14."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Prerequisites (continued)\n",
    "\n",
    "- You have predictions, model outputs that are stored somewhere. They are frozen, just like the input observations. Perhaps they are “small” and may conveniently be stored in a git repo. Or perhaps they are big and are more conveniently stored in S3 or table. If the table contains additional rows, be sure your ReadMe or review-request shows how to query just the rows that matter for this review.\n",
    "- You think the model is mature enough for review. Consider running it through a brief code review beforehand. \n",
    "- Consider doing a dry run, where Author pretends he is Reviewer and verifies the task is feasible."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Author process\n",
    "\n",
    "- Pick a reviewer. That’s a reviewer, a single reviewer, just one, as there will be some effort involved. Count not to two. We can invite more to the party later.\n",
    "\n",
    "- Create a new git feature branch (along with a jira ticket) for this review. Not for code development. Just for review.\n",
    "\n",
    "- Commit and push a file full of review-specific instructions to the reviewer, with a name like doc/2020-04-14-review-request.md, or in toplevel dir, or whatever is convenient for your repo. Invite the reviewer to tweak an aspect of training or prediction, something that won’t take days of compute time. (Consider testing it yourself! For “big” models, consider producing model plus toy_model, which processes small input in less than an hour. There’s lots of things the reviewer might do – your job is to make it so easy that many of them are quickly accomplished.)\n",
    "\n",
    "- Push to Bitbucket and send your reviewer a PR in the usual way. Make no further edits. This branch is no longer yours – it belongs exclusively to the single reviewer you nominated."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Reviewer process (continued)\n",
    "\n",
    "- Checkout the feature / review branch.\n",
    "- Create and commit an empty file with a name like doc/2020-04-16-review-result.md.\n",
    "- Read the Objective, found in README.md or wherever the review-request explains it may be found. Add a sentence or two to your review-result file, describing whether the Objective is clear and seems relevant to stakeholders.\n",
    "- Read the frozen model inputs, a few individual records plus stat summaries that Author helpfully made it easy to view. Write a sentence or two describing whether inputs seem to match reality, and match the Objective.\n",
    "- Read the frozen model outputs. Write a sentence or two describing whether they seem to match reality, and match the Objective. For each of these three, a simple “makes sense,   yes, I agree” will suffice."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Reviewer process (continued)\n",
    "\n",
    "- Add a sentence or two describing transformation(s) done during ETL, and whether they seem reasonable. You should be starting with copy-n-paste of a sentence the Author helpfully put in the ReadMe.\n",
    "- Read the model code, if you like, and append comments to the review-result file. This part is optional – code review should have been handled prior to model review, perhaps involving same reviewer. The code must respect frozen inputs and outputs, leaving them untouched. It must be able to send reviewer’s output to a new S3 object, table, or similar.\n",
    "- Retrain the model from frozen inputs, or at least retrain toy_model. Timebox to one hour running time, and add a paragraph to review-result. Write “took too long” if model or toy_model get stuck at this stage.\n",
    "- Reproduce the frozen model outputs. Timebox to one hour running time, and add a paragraph to review-result."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Reviewer process (continued)\n",
    "- Read the review-request instructions and tweak the model in the suggested way. Timebox to one hour, and add a paragraph to review-result.\n",
    "- Tweak training or prediction in a way you find interesting. Timebox to one hour, and add a paragraph to review-result.\n",
    "- Pick an example prediction error, or a summary description of how the error is distributed. Add a paragraph describing why the error makes sense, or does not.\n",
    "- Revisit the Objective. Add a paragraph relating it to the model. Describe Next Steps, things the Reviewer feels offer opportunity for improvement, based on what we learned during review.\n",
    "- Do final commit on review-result, push branch. Invite others / comment within the PR if you wish. Click Approve on the PR, and click Merge down to develop."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## ZOMG It's down!!!\n",
    "\n",
    "\n",
    "<img src=\"../images/casey-horner-y7jrFSlVZAQ-unsplash-cropped.jpg\" width=720>\n",
    "\n",
    "\n",
    "<span style=\"font-size: small\">Photo by <a href=\"https://unsplash.com/@mischievous_penguins?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText\">Casey Horner</a> on <a href=\"https://unsplash.com/s/photos/ship?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText\">Unsplash</a></span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Define healthchecks and alerts\n",
    "\n",
    "<span style=\"display: inline-block; height:25px; width:720px\"></span>\n",
    "\n",
    "<div style=\"display: flex;\">\n",
    "<div style=\"height:250px; width:25%\"></div>\n",
    "<div>\n",
    "<img src=\"../images/terrel-system-diagram.png\" width=\"1000\">\n",
    "    </div>\n",
    "</div>\n",
    "\n",
    "<span style=\"font-size:small\">Figure by Andy R. Terrel</span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Use model servers\n",
    "<span style=\"display: inline-block; height:25px; width:720px\"></span>\n",
    "\n",
    "<div style=\"display: flex;\">\n",
    "<div style=\"height:250px; width:25%\"></div>\n",
    "<div>\n",
    "    <img src=\"../images/bentoml-readme-header.jpeg\" width=750>\n",
    "    <img style=\"background: black;padding: 5px\" src=\"../images/MLflow-logo-final-white-TM.png\" width=750>\n",
    "    <img src=\"../images/redis-ai.png\" width=750>\n",
    "    </div>\n",
    "</div>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Finally time to start again!\n",
    "\n",
    "<span style=\"display: inline-block; height:25px; width:720px\"></span>\n",
    "\n",
    "<div style=\"display: flex;\">\n",
    "<div style=\"height:250px; width:25%\"></div>\n",
    "<div>\n",
    "<img src=\"../images/whoisbenjamin-ApJp5Nk24a0-unsplash.jpg\" width=\"950\">\n",
    "    </div>\n",
    "</div>\n",
    "\n",
    "<span style=\"font-size: small\">Photo by <a href=\"https://unsplash.com/@whoisbenjamin?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText\">@whoisbenjamin</a> on <a href=\"https://unsplash.com/s/photos/ships-indian-ocean?utm_source=unsplash&amp;utm_medium=referral&amp;utm_content=creditCopyText\">Unsplash</a></span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}