test-image-open-payments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: Pillow in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (7.1.1)\n",
      "\u001b[33mWARNING: You are using pip version 19.2.3, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: nb_black in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (1.0.7)\n",
      "Requirement already satisfied: black>='19.3'; python_version >= \"3.6\" in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from nb_black) (19.10b0)\n",
      "Requirement already satisfied: ipython in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from nb_black) (7.13.0)\n",
      "Requirement already satisfied: attrs>=18.1.0 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from black>='19.3'; python_version >= \"3.6\"->nb_black) (19.3.0)\n",
      "Requirement already satisfied: regex in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from black>='19.3'; python_version >= \"3.6\"->nb_black) (2020.4.4)\n",
      "Requirement already satisfied: typed-ast>=1.4.0 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from black>='19.3'; python_version >= \"3.6\"->nb_black) (1.4.1)\n",
      "Requirement already satisfied: click>=6.5 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from black>='19.3'; python_version >= \"3.6\"->nb_black) (7.1.1)\n",
      "Requirement already satisfied: pathspec<1,>=0.6 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from black>='19.3'; python_version >= \"3.6\"->nb_black) (0.8.0)\n",
      "Requirement already satisfied: appdirs in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from black>='19.3'; python_version >= \"3.6\"->nb_black) (1.4.3)\n",
      "Requirement already satisfied: toml>=0.9.4 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from black>='19.3'; python_version >= \"3.6\"->nb_black) (0.10.0)\n",
      "Requirement already satisfied: pickleshare in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from ipython->nb_black) (0.7.5)\n",
      "Requirement already satisfied: pexpect; sys_platform != \"win32\" in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from ipython->nb_black) (4.8.0)\n",
      "Requirement already satisfied: pygments in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from ipython->nb_black) (2.6.1)\n",
      "Requirement already satisfied: decorator in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from ipython->nb_black) (4.4.2)\n",
      "Requirement already satisfied: backcall in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from ipython->nb_black) (0.1.0)\n",
      "Requirement already satisfied: jedi>=0.10 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from ipython->nb_black) (0.16.0)\n",
      "Requirement already satisfied: traitlets>=4.2 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from ipython->nb_black) (4.3.3)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from ipython->nb_black) (3.0.4)\n",
      "Requirement already satisfied: setuptools>=18.5 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from ipython->nb_black) (41.2.0)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from pexpect; sys_platform != \"win32\"->ipython->nb_black) (0.6.0)\n",
      "Requirement already satisfied: parso>=0.5.2 in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from jedi>=0.10->ipython->nb_black) (0.6.2)\n",
      "Requirement already satisfied: ipython-genutils in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from traitlets>=4.2->ipython->nb_black) (0.2.0)\n",
      "Requirement already satisfied: six in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from traitlets>=4.2->ipython->nb_black) (1.14.0)\n",
      "Requirement already satisfied: wcwidth in ./.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython->nb_black) (0.1.9)\n",
      "\u001b[33mWARNING: You are using pip version 19.2.3, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 1;\n",
       "                var nbb_unformatted_code = \"# Pre requisites for this notebook\\n!pip install Pillow\\n!pip install nb_black\\n%load_ext nb_black\";\n",
       "                var nbb_formatted_code = \"# Pre requisites for this notebook\\n!pip install Pillow\\n!pip install nb_black\\n%load_ext nb_black\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Pre requisites for this notebook\n",
    "!pip install Pillow\n",
    "!pip install nb_black\n",
    "%load_ext nb_black"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 2;\n",
       "                var nbb_unformatted_code = \"import os\\nfrom requests import get\\nfrom pathlib import Path\\nimport gc\\nimport tensorflow as tf\\nfrom concurrent.futures import ProcessPoolExecutor as PoolExecutor\\n\\nfrom tensorflow.keras.callbacks import EarlyStopping\\nfrom tensorflow_addons.optimizers import RectifiedAdam, Lookahead\\nfrom tensorflow_addons.activations import mish\\n\\nimport pandas as pd\\nimport numpy as np\\nfrom sklearn.metrics import roc_auc_score\\n\\nnp.random.seed(0)\\n\\nfrom PIL import Image, ImageDraw, ImageFont\\nfrom matplotlib.pyplot import imshow\\nimport matplotlib.pyplot as plt\\n\\n%matplotlib inline\";\n",
       "                var nbb_formatted_code = \"import os\\nfrom requests import get\\nfrom pathlib import Path\\nimport gc\\nimport tensorflow as tf\\nfrom concurrent.futures import ProcessPoolExecutor as PoolExecutor\\n\\nfrom tensorflow.keras.callbacks import EarlyStopping\\nfrom tensorflow_addons.optimizers import RectifiedAdam, Lookahead\\nfrom tensorflow_addons.activations import mish\\n\\nimport pandas as pd\\nimport numpy as np\\nfrom sklearn.metrics import roc_auc_score\\n\\nnp.random.seed(0)\\n\\nfrom PIL import Image, ImageDraw, ImageFont\\nfrom matplotlib.pyplot import imshow\\nimport matplotlib.pyplot as plt\\n\\n%matplotlib inline\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "from requests import get\n",
    "from pathlib import Path\n",
    "import gc\n",
    "import tensorflow as tf\n",
    "from concurrent.futures import ProcessPoolExecutor as PoolExecutor\n",
    "\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow_addons.optimizers import RectifiedAdam, Lookahead\n",
    "from tensorflow_addons.activations import mish\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "np.random.seed(0)\n",
    "\n",
    "from PIL import Image, ImageDraw, ImageFont\n",
    "from matplotlib.pyplot import imshow\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 3;\n",
       "                var nbb_unformatted_code = \"def download(url, out, force=False, verify=True):\\n    out.parent.mkdir(parents=True, exist_ok=True)\\n    if force:\\n        print(f\\\"Removing file at {str(out)}\\\")\\n        out.unlink()\\n\\n    if out.exists():\\n        print(\\\"File already exists.\\\")\\n        return\\n    print(f\\\"Downloading {url} at {str(out)} ...\\\")\\n    # open in binary mode\\n    with out.open(mode=\\\"wb\\\") as file:\\n        # get request\\n        response = get(url, verify=verify)\\n        for chunk in response.iter_content(100000):\\n            # write to file\\n            file.write(chunk)\";\n",
       "                var nbb_formatted_code = \"def download(url, out, force=False, verify=True):\\n    out.parent.mkdir(parents=True, exist_ok=True)\\n    if force:\\n        print(f\\\"Removing file at {str(out)}\\\")\\n        out.unlink()\\n\\n    if out.exists():\\n        print(\\\"File already exists.\\\")\\n        return\\n    print(f\\\"Downloading {url} at {str(out)} ...\\\")\\n    # open in binary mode\\n    with out.open(mode=\\\"wb\\\") as file:\\n        # get request\\n        response = get(url, verify=verify)\\n        for chunk in response.iter_content(100000):\\n            # write to file\\n            file.write(chunk)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def download(url, out, force=False, verify=True):\n",
    "    out.parent.mkdir(parents=True, exist_ok=True)\n",
    "    if force:\n",
    "        print(f\"Removing file at {str(out)}\")\n",
    "        out.unlink()\n",
    "\n",
    "    if out.exists():\n",
    "        print(\"File already exists.\")\n",
    "        return\n",
    "    print(f\"Downloading {url} at {str(out)} ...\")\n",
    "    # open in binary mode\n",
    "    with out.open(mode=\"wb\") as file:\n",
    "        # get request\n",
    "        response = get(url, verify=verify)\n",
    "        for chunk in response.iter_content(100000):\n",
    "            # write to file\n",
    "            file.write(chunk)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download open-payments dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File already exists.\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 4;\n",
       "                var nbb_unformatted_code = \"# url = \\\"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data\\\"\\n# url_test = \\\"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.test\\\"\\nfont_url = \\\"https://ff.static.1001fonts.net/r/o/roboto-condensed.regular.ttf\\\"\\n\\ndataset_name = \\\"open-payments\\\"\\nout = Path(os.getcwd()) / f\\\"data/{dataset_name}/train_bench.csv\\\"\\n# out_test = Path(os.getcwd()) / f\\\"data/{dataset_name}_test.csv\\\"\\nout_font = Path(os.getcwd()) / f\\\"RobotoCondensed-Regular.ttf\\\"\\n\\n# download(url, out)\\n# download(url_test, out_test)\\ndownload(font_url, out_font)\";\n",
       "                var nbb_formatted_code = \"# url = \\\"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data\\\"\\n# url_test = \\\"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.test\\\"\\nfont_url = \\\"https://ff.static.1001fonts.net/r/o/roboto-condensed.regular.ttf\\\"\\n\\ndataset_name = \\\"open-payments\\\"\\nout = Path(os.getcwd()) / f\\\"data/{dataset_name}/train_bench.csv\\\"\\n# out_test = Path(os.getcwd()) / f\\\"data/{dataset_name}_test.csv\\\"\\nout_font = Path(os.getcwd()) / f\\\"RobotoCondensed-Regular.ttf\\\"\\n\\n# download(url, out)\\n# download(url_test, out_test)\\ndownload(font_url, out_font)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data\"\n",
    "# url_test = \"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.test\"\n",
    "font_url = \"https://ff.static.1001fonts.net/r/o/roboto-condensed.regular.ttf\"\n",
    "\n",
    "dataset_name = \"open-payments\"\n",
    "out = Path(os.getcwd()) / f\"data/{dataset_name}/train_bench.csv\"\n",
    "# out_test = Path(os.getcwd()) / f\"data/{dataset_name}_test.csv\"\n",
    "out_font = Path(os.getcwd()) / f\"RobotoCondensed-Regular.ttf\"\n",
    "\n",
    "# download(url, out)\n",
    "# download(url_test, out_test)\n",
    "download(font_url, out_font)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load open-payments as a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(73354, 7)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"target = \\\"status\\\"\\ntrain = pd.read_csv(out, low_memory=False)\\nprint(train.shape)\";\n",
       "                var nbb_formatted_code = \"target = \\\"status\\\"\\ntrain = pd.read_csv(out, low_memory=False)\\nprint(train.shape)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "target = \"status\"\n",
    "train = pd.read_csv(out, low_memory=False)\n",
    "print(train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"train_indices = train[train.Set == \\\"train\\\"].index.values\\nnp.random.shuffle(train_indices)\\nvalid_indices = train[train.Set == \\\"valid\\\"].index.values\\ntest_indices = train[train.Set == \\\"test\\\"].index.values\";\n",
       "                var nbb_formatted_code = \"train_indices = train[train.Set == \\\"train\\\"].index.values\\nnp.random.shuffle(train_indices)\\nvalid_indices = train[train.Set == \\\"valid\\\"].index.values\\ntest_indices = train[train.Set == \\\"test\\\"].index.values\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_indices = train[train.Set == \"train\"].index.values\n",
    "np.random.shuffle(train_indices)\n",
    "valid_indices = train[train.Set == \"valid\"].index.values\n",
    "test_indices = train[train.Set == \"test\"].index.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(73354, 1)\n",
      "(73354, 5)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 7;\n",
       "                var nbb_unformatted_code = \"from sklearn import preprocessing\\n\\n# label_encoder object knows how to understand word labels.\\nlabel_encoder = preprocessing.LabelEncoder()\\n\\n# Encode labels in column 'species'.\\nY = (\\n    label_encoder.fit_transform(train[[target]].values.reshape(-1))\\n    .reshape(-1, 1)\\n    .astype(\\\"uint8\\\")\\n)\\nprint(Y.shape)\\n\\nX = train.drop(columns=[target, \\\"Set\\\"]).values.astype(\\\"str\\\")\\nprint(X.shape)\\n\\ndel train\\ngc.collect()\";\n",
       "                var nbb_formatted_code = \"from sklearn import preprocessing\\n\\n# label_encoder object knows how to understand word labels.\\nlabel_encoder = preprocessing.LabelEncoder()\\n\\n# Encode labels in column 'species'.\\nY = (\\n    label_encoder.fit_transform(train[[target]].values.reshape(-1))\\n    .reshape(-1, 1)\\n    .astype(\\\"uint8\\\")\\n)\\nprint(Y.shape)\\n\\nX = train.drop(columns=[target, \\\"Set\\\"]).values.astype(\\\"str\\\")\\nprint(X.shape)\\n\\ndel train\\ngc.collect()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "# label_encoder object knows how to understand word labels.\n",
    "label_encoder = preprocessing.LabelEncoder()\n",
    "\n",
    "# Encode labels in column 'species'.\n",
    "Y = (\n",
    "    label_encoder.fit_transform(train[[target]].values.reshape(-1))\n",
    "    .reshape(-1, 1)\n",
    "    .astype(\"uint8\")\n",
    ")\n",
    "print(Y.shape)\n",
    "\n",
    "X = train.drop(columns=[target, \"Set\"]).values.astype(\"str\")\n",
    "print(X.shape)\n",
    "\n",
    "del train\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 8;\n",
       "                var nbb_unformatted_code = \"# https://he-arc.github.io/livre-python/pillow/index.html#methodes-de-dessin\\n# https://stackoverflow.com/questions/26649716/how-to-show-pil-image-in-ipython-notebook\\n# https://stackoverflow.com/questions/384759/how-to-convert-a-pil-image-into-a-numpy-array\\n# line = np.array(pic, dtypes=\\\"uint8\\\")\\n# from https://arxiv.org/pdf/1902.02160.pdf page 2\";\n",
       "                var nbb_formatted_code = \"# https://he-arc.github.io/livre-python/pillow/index.html#methodes-de-dessin\\n# https://stackoverflow.com/questions/26649716/how-to-show-pil-image-in-ipython-notebook\\n# https://stackoverflow.com/questions/384759/how-to-convert-a-pil-image-into-a-numpy-array\\n# line = np.array(pic, dtypes=\\\"uint8\\\")\\n# from https://arxiv.org/pdf/1902.02160.pdf page 2\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# https://he-arc.github.io/livre-python/pillow/index.html#methodes-de-dessin\n",
    "# https://stackoverflow.com/questions/26649716/how-to-show-pil-image-in-ipython-notebook\n",
    "# https://stackoverflow.com/questions/384759/how-to-convert-a-pil-image-into-a-numpy-array\n",
    "# line = np.array(pic, dtypes=\"uint8\")\n",
    "# from https://arxiv.org/pdf/1902.02160.pdf page 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 9;\n",
       "                var nbb_unformatted_code = \"WHITE = (255, 255, 255)\\nBLACK = (0, 0, 0)\\n\\n\\ndef word_to_square_image(text, size, cut_length=None):\\n\\n    truncated = text[:cut_length] if cut_length is not None else text\\n    max_x = np.ceil(np.sqrt(len(truncated))).astype(\\\"int\\\")\\n    character_size = np.floor(size / max_x).astype(\\\"int\\\")\\n    padding = np.floor((size - (max_x * character_size)) / 2).astype(\\\"int\\\")\\n    # Do we need pt to px conversion ? Seems like not\\n    # font_size =  int(np.floor(character_size*0.75))\\n    font_size = character_size\\n\\n    fnt = ImageFont.truetype(out_font.as_posix(), font_size)\\n    image = Image.new(\\\"RGB\\\", (size, size), BLACK)\\n    # Obtention du contexte graphique\\n    draw = ImageDraw.Draw(image)\\n    x = 0\\n    y = 0\\n    for letter in text:\\n        draw.text(\\n            (padding + x * character_size, padding + y * character_size),\\n            letter,\\n            font=fnt,\\n            fill=WHITE,\\n        )\\n        if x + 1 < max_x:\\n            x += 1\\n        else:\\n            y += 1\\n            x = 0\\n    return np.array(image)\";\n",
       "                var nbb_formatted_code = \"WHITE = (255, 255, 255)\\nBLACK = (0, 0, 0)\\n\\n\\ndef word_to_square_image(text, size, cut_length=None):\\n\\n    truncated = text[:cut_length] if cut_length is not None else text\\n    max_x = np.ceil(np.sqrt(len(truncated))).astype(\\\"int\\\")\\n    character_size = np.floor(size / max_x).astype(\\\"int\\\")\\n    padding = np.floor((size - (max_x * character_size)) / 2).astype(\\\"int\\\")\\n    # Do we need pt to px conversion ? Seems like not\\n    # font_size =  int(np.floor(character_size*0.75))\\n    font_size = character_size\\n\\n    fnt = ImageFont.truetype(out_font.as_posix(), font_size)\\n    image = Image.new(\\\"RGB\\\", (size, size), BLACK)\\n    # Obtention du contexte graphique\\n    draw = ImageDraw.Draw(image)\\n    x = 0\\n    y = 0\\n    for letter in text:\\n        draw.text(\\n            (padding + x * character_size, padding + y * character_size),\\n            letter,\\n            font=fnt,\\n            fill=WHITE,\\n        )\\n        if x + 1 < max_x:\\n            x += 1\\n        else:\\n            y += 1\\n            x = 0\\n    return np.array(image)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "WHITE = (255, 255, 255)\n",
    "BLACK = (0, 0, 0)\n",
    "\n",
    "\n",
    "def word_to_square_image(text, size, cut_length=None):\n",
    "\n",
    "    truncated = text[:cut_length] if cut_length is not None else text\n",
    "    max_x = np.ceil(np.sqrt(len(truncated))).astype(\"int\")\n",
    "    character_size = np.floor(size / max_x).astype(\"int\")\n",
    "    padding = np.floor((size - (max_x * character_size)) / 2).astype(\"int\")\n",
    "    # Do we need pt to px conversion ? Seems like not\n",
    "    # font_size =  int(np.floor(character_size*0.75))\n",
    "    font_size = character_size\n",
    "\n",
    "    fnt = ImageFont.truetype(out_font.as_posix(), font_size)\n",
    "    image = Image.new(\"RGB\", (size, size), BLACK)\n",
    "    # Obtention du contexte graphique\n",
    "    draw = ImageDraw.Draw(image)\n",
    "    x = 0\n",
    "    y = 0\n",
    "    for letter in text:\n",
    "        draw.text(\n",
    "            (padding + x * character_size, padding + y * character_size),\n",
    "            letter,\n",
    "            font=fnt,\n",
    "            fill=WHITE,\n",
    "        )\n",
    "        if x + 1 < max_x:\n",
    "            x += 1\n",
    "        else:\n",
    "            y += 1\n",
    "            x = 0\n",
    "    return np.array(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 10;\n",
       "                var nbb_unformatted_code = \"def text_to_square_image(features, image_size=299, cut_length=None):\\n    square_nb = np.ceil(np.sqrt(len(features))).astype(\\\"int\\\")\\n    word_size = np.floor(image_size / square_nb).astype(\\\"int\\\")\\n    max_features = len(features)\\n    padding = np.floor((image_size - square_nb * word_size) / 2).astype(\\\"int\\\")\\n    result_image = np.zeros((image_size, image_size, 3), dtype=\\\"uint8\\\")\\n    results = []\\n    i_feature = 0\\n    for x in range(0, square_nb):\\n        if i_feature is None:\\n            break\\n        for y in range(0, square_nb):\\n            i_feature = x * (square_nb) + y\\n            if i_feature >= max_features:\\n                i_feature = None\\n                break\\n            x_pos = x * word_size + padding\\n            y_pos = y * word_size + padding\\n            result_image[\\n                x_pos : x_pos + word_size, y_pos : y_pos + word_size\\n            ] = word_to_square_image(\\n                features[i_feature], size=word_size, cut_length=cut_length\\n            )\\n    return result_image\";\n",
       "                var nbb_formatted_code = \"def text_to_square_image(features, image_size=299, cut_length=None):\\n    square_nb = np.ceil(np.sqrt(len(features))).astype(\\\"int\\\")\\n    word_size = np.floor(image_size / square_nb).astype(\\\"int\\\")\\n    max_features = len(features)\\n    padding = np.floor((image_size - square_nb * word_size) / 2).astype(\\\"int\\\")\\n    result_image = np.zeros((image_size, image_size, 3), dtype=\\\"uint8\\\")\\n    results = []\\n    i_feature = 0\\n    for x in range(0, square_nb):\\n        if i_feature is None:\\n            break\\n        for y in range(0, square_nb):\\n            i_feature = x * (square_nb) + y\\n            if i_feature >= max_features:\\n                i_feature = None\\n                break\\n            x_pos = x * word_size + padding\\n            y_pos = y * word_size + padding\\n            result_image[\\n                x_pos : x_pos + word_size, y_pos : y_pos + word_size\\n            ] = word_to_square_image(\\n                features[i_feature], size=word_size, cut_length=cut_length\\n            )\\n    return result_image\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def text_to_square_image(features, image_size=299, cut_length=None):\n",
    "    square_nb = np.ceil(np.sqrt(len(features))).astype(\"int\")\n",
    "    word_size = np.floor(image_size / square_nb).astype(\"int\")\n",
    "    max_features = len(features)\n",
    "    padding = np.floor((image_size - square_nb * word_size) / 2).astype(\"int\")\n",
    "    result_image = np.zeros((image_size, image_size, 3), dtype=\"uint8\")\n",
    "    results = []\n",
    "    i_feature = 0\n",
    "    for x in range(0, square_nb):\n",
    "        if i_feature is None:\n",
    "            break\n",
    "        for y in range(0, square_nb):\n",
    "            i_feature = x * (square_nb) + y\n",
    "            if i_feature >= max_features:\n",
    "                i_feature = None\n",
    "                break\n",
    "            x_pos = x * word_size + padding\n",
    "            y_pos = y * word_size + padding\n",
    "            result_image[\n",
    "                x_pos : x_pos + word_size, y_pos : y_pos + word_size\n",
    "            ] = word_to_square_image(\n",
    "                features[i_feature], size=word_size, cut_length=cut_length\n",
    "            )\n",
    "    return result_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 11;\n",
       "                var nbb_unformatted_code = \"def preprocess_data(data, map_func):\\n    preprocessed_df = None\\n    with PoolExecutor() as executor:\\n        preprocessed_df = np.stack(list(executor.map(map_func, data)), axis=0)\\n    print(preprocessed_df.shape)\\n    print(preprocessed_df.nbytes / (1024 * 1024))  # Memory size in RAM\\n    gc.collect()\\n    return preprocessed_df\";\n",
       "                var nbb_formatted_code = \"def preprocess_data(data, map_func):\\n    preprocessed_df = None\\n    with PoolExecutor() as executor:\\n        preprocessed_df = np.stack(list(executor.map(map_func, data)), axis=0)\\n    print(preprocessed_df.shape)\\n    print(preprocessed_df.nbytes / (1024 * 1024))  # Memory size in RAM\\n    gc.collect()\\n    return preprocessed_df\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def preprocess_data(data, map_func):\n",
    "    preprocessed_df = None\n",
    "    with PoolExecutor() as executor:\n",
    "        preprocessed_df = np.stack(list(executor.map(map_func, data)), axis=0)\n",
    "    print(preprocessed_df.shape)\n",
    "    print(preprocessed_df.nbytes / (1024 * 1024))  # Memory size in RAM\n",
    "    gc.collect()\n",
    "    return preprocessed_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 12;\n",
       "                var nbb_unformatted_code = \"def fixed_text_to_square_image(values):\\n    return text_to_square_image(values, image_size=96, cut_length=None)\";\n",
       "                var nbb_formatted_code = \"def fixed_text_to_square_image(values):\\n    return text_to_square_image(values, image_size=96, cut_length=None)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def fixed_text_to_square_image(values):\n",
    "    return text_to_square_image(values, image_size=96, cut_length=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 13;\n",
       "                var nbb_unformatted_code = \"from tensorflow.keras.utils import to_categorical\\nfrom concurrent.futures import ProcessPoolExecutor as PoolExecutor\\n\\n\\nclass TabularToImagesDataset:\\n    def __init__(self, values, target, func, prefetch=1024):\\n        self.values = values\\n        self.target = to_categorical(target.reshape(-1))\\n        assert target.shape[0] == self.values.shape[0]\\n        self.current = -1\\n        self.max_prefetch = -1\\n        self.prefetch_nb = prefetch\\n        self.func = func\\n        self.ready = None\\n\\n    def __iter__(self):\\n        # print(\\\"Calling __iter__\\\")\\n        self.current = -1\\n        self.max_prefetch = -1\\n        return self\\n\\n    def __call__(self):\\n        # print(\\\"Calling __call__\\\")\\n        self.current = -1\\n        self.max_prefetch = -1\\n        return self\\n\\n    def __next__(self):\\n        return self.next()\\n\\n    def __len__(self):\\n        return self.values.shape[0]\\n\\n    def prefetch(self):\\n        if self.ready is not None and self.ready.shape[0] == self.values.shape[0]:\\n            return\\n        # print(\\\"HERE\\\")\\n        self.max_prefetch = min(self.current + self.prefetch_nb, self.values.shape[0])\\n        # if self.current == self.max_prefetch:\\n\\n        with PoolExecutor() as executor:\\n            self.ready = np.stack(\\n                list(\\n                    executor.map(\\n                        self.func, self.values[self.current : self.max_prefetch]\\n                    )\\n                ),\\n                axis=0,\\n            )\\n        return\\n\\n    def next(self):\\n        self.current += 1\\n        # print(self.current)\\n        if self.current >= self.values.shape[0]:\\n            raise StopIteration()\\n        # self.current = 0\\n        # self.max_prefetch = -1\\n        if self.current >= self.max_prefetch:\\n            # print(\\\"Will prefetch\\\")\\n            self.prefetch()\\n            # print(self.ready.shape)\\n        return (\\n            self.ready[self.current % self.prefetch_nb],\\n            # text_to_square_image(self.values[self.current]),\\n            self.target[self.current],\\n        )\";\n",
       "                var nbb_formatted_code = \"from tensorflow.keras.utils import to_categorical\\nfrom concurrent.futures import ProcessPoolExecutor as PoolExecutor\\n\\n\\nclass TabularToImagesDataset:\\n    def __init__(self, values, target, func, prefetch=1024):\\n        self.values = values\\n        self.target = to_categorical(target.reshape(-1))\\n        assert target.shape[0] == self.values.shape[0]\\n        self.current = -1\\n        self.max_prefetch = -1\\n        self.prefetch_nb = prefetch\\n        self.func = func\\n        self.ready = None\\n\\n    def __iter__(self):\\n        # print(\\\"Calling __iter__\\\")\\n        self.current = -1\\n        self.max_prefetch = -1\\n        return self\\n\\n    def __call__(self):\\n        # print(\\\"Calling __call__\\\")\\n        self.current = -1\\n        self.max_prefetch = -1\\n        return self\\n\\n    def __next__(self):\\n        return self.next()\\n\\n    def __len__(self):\\n        return self.values.shape[0]\\n\\n    def prefetch(self):\\n        if self.ready is not None and self.ready.shape[0] == self.values.shape[0]:\\n            return\\n        # print(\\\"HERE\\\")\\n        self.max_prefetch = min(self.current + self.prefetch_nb, self.values.shape[0])\\n        # if self.current == self.max_prefetch:\\n\\n        with PoolExecutor() as executor:\\n            self.ready = np.stack(\\n                list(\\n                    executor.map(\\n                        self.func, self.values[self.current : self.max_prefetch]\\n                    )\\n                ),\\n                axis=0,\\n            )\\n        return\\n\\n    def next(self):\\n        self.current += 1\\n        # print(self.current)\\n        if self.current >= self.values.shape[0]:\\n            raise StopIteration()\\n        # self.current = 0\\n        # self.max_prefetch = -1\\n        if self.current >= self.max_prefetch:\\n            # print(\\\"Will prefetch\\\")\\n            self.prefetch()\\n            # print(self.ready.shape)\\n        return (\\n            self.ready[self.current % self.prefetch_nb],\\n            # text_to_square_image(self.values[self.current]),\\n            self.target[self.current],\\n        )\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tensorflow.keras.utils import to_categorical\n",
    "from concurrent.futures import ProcessPoolExecutor as PoolExecutor\n",
    "\n",
    "\n",
    "class TabularToImagesDataset:\n",
    "    def __init__(self, values, target, func, prefetch=1024):\n",
    "        self.values = values\n",
    "        self.target = to_categorical(target.reshape(-1))\n",
    "        assert target.shape[0] == self.values.shape[0]\n",
    "        self.current = -1\n",
    "        self.max_prefetch = -1\n",
    "        self.prefetch_nb = prefetch\n",
    "        self.func = func\n",
    "        self.ready = None\n",
    "\n",
    "    def __iter__(self):\n",
    "        # print(\"Calling __iter__\")\n",
    "        self.current = -1\n",
    "        self.max_prefetch = -1\n",
    "        return self\n",
    "\n",
    "    def __call__(self):\n",
    "        # print(\"Calling __call__\")\n",
    "        self.current = -1\n",
    "        self.max_prefetch = -1\n",
    "        return self\n",
    "\n",
    "    def __next__(self):\n",
    "        return self.next()\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.values.shape[0]\n",
    "\n",
    "    def prefetch(self):\n",
    "        if self.ready is not None and self.ready.shape[0] == self.values.shape[0]:\n",
    "            return\n",
    "        # print(\"HERE\")\n",
    "        self.max_prefetch = min(self.current + self.prefetch_nb, self.values.shape[0])\n",
    "        # if self.current == self.max_prefetch:\n",
    "\n",
    "        with PoolExecutor() as executor:\n",
    "            self.ready = np.stack(\n",
    "                list(\n",
    "                    executor.map(\n",
    "                        self.func, self.values[self.current : self.max_prefetch]\n",
    "                    )\n",
    "                ),\n",
    "                axis=0,\n",
    "            )\n",
    "        return\n",
    "\n",
    "    def next(self):\n",
    "        self.current += 1\n",
    "        # print(self.current)\n",
    "        if self.current >= self.values.shape[0]:\n",
    "            raise StopIteration()\n",
    "        # self.current = 0\n",
    "        # self.max_prefetch = -1\n",
    "        if self.current >= self.max_prefetch:\n",
    "            # print(\"Will prefetch\")\n",
    "            self.prefetch()\n",
    "            # print(self.ready.shape)\n",
    "        return (\n",
    "            self.ready[self.current % self.prefetch_nb],\n",
    "            # text_to_square_image(self.values[self.current]),\n",
    "            self.target[self.current],\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 14;\n",
       "                var nbb_unformatted_code = \"def build_tf_dataset(X_values, Y_values, image_size, fix_func, prefetch, batch_size):\\n    gen = TabularToImagesDataset(\\n        X_values, Y_values, fix_func, prefetch=prefetch * batch_size,\\n    )\\n    if prefetch * batch_size > X_values.shape[0]:\\n        prefetch = np.ceil(X_values.shape[0] / batch_size).astype(\\\"int\\\")\\n\\n    dataset = tf.data.Dataset.from_generator(\\n        gen,\\n        (tf.uint8, tf.uint8),\\n        (tf.TensorShape([image_size, image_size, 3]), tf.TensorShape([2])),\\n    )\\n\\n    dataset = dataset.repeat().batch(batch_size)\\n    return dataset.prefetch(prefetch)\";\n",
       "                var nbb_formatted_code = \"def build_tf_dataset(X_values, Y_values, image_size, fix_func, prefetch, batch_size):\\n    gen = TabularToImagesDataset(\\n        X_values, Y_values, fix_func, prefetch=prefetch * batch_size,\\n    )\\n    if prefetch * batch_size > X_values.shape[0]:\\n        prefetch = np.ceil(X_values.shape[0] / batch_size).astype(\\\"int\\\")\\n\\n    dataset = tf.data.Dataset.from_generator(\\n        gen,\\n        (tf.uint8, tf.uint8),\\n        (tf.TensorShape([image_size, image_size, 3]), tf.TensorShape([2])),\\n    )\\n\\n    dataset = dataset.repeat().batch(batch_size)\\n    return dataset.prefetch(prefetch)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def build_tf_dataset(X_values, Y_values, image_size, fix_func, prefetch, batch_size):\n",
    "    gen = TabularToImagesDataset(\n",
    "        X_values, Y_values, fix_func, prefetch=prefetch * batch_size,\n",
    "    )\n",
    "    if prefetch * batch_size > X_values.shape[0]:\n",
    "        prefetch = np.ceil(X_values.shape[0] / batch_size).astype(\"int\")\n",
    "\n",
    "    dataset = tf.data.Dataset.from_generator(\n",
    "        gen,\n",
    "        (tf.uint8, tf.uint8),\n",
    "        (tf.TensorShape([image_size, image_size, 3]), tf.TensorShape([2])),\n",
    "    )\n",
    "\n",
    "    dataset = dataset.repeat().batch(batch_size)\n",
    "    return dataset.prefetch(prefetch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 15;\n",
       "                var nbb_unformatted_code = \"epochs_1 = 2\\nepochs_2 = 200\\nimage_size = 96\\ncut_length = None\\nBATCH_SIZE = 256\\nPREFETCH = 10000  # 40\\n# SHUFFLE_BUFFER_SIZE = 10000\\npatience = 5\\n\\n\\ndef fixed_text_to_square_image(values):\\n    return text_to_square_image(values, image_size=image_size, cut_length=cut_length)\\n\\n\\nsteps_per_epoch = np.ceil(train_indices.shape[0] / BATCH_SIZE)\\nsteps_per_epoch_val = np.ceil(valid_indices.shape[0] / BATCH_SIZE)\\n\\ndataset = build_tf_dataset(\\n    X[train_indices],\\n    Y[train_indices],\\n    image_size,\\n    fixed_text_to_square_image,\\n    PREFETCH,\\n    BATCH_SIZE,\\n)\\n\\ndataset_valid = build_tf_dataset(\\n    X[valid_indices],\\n    Y[valid_indices],\\n    image_size,\\n    fixed_text_to_square_image,\\n    PREFETCH,\\n    BATCH_SIZE,\\n)\";\n",
       "                var nbb_formatted_code = \"epochs_1 = 2\\nepochs_2 = 200\\nimage_size = 96\\ncut_length = None\\nBATCH_SIZE = 256\\nPREFETCH = 10000  # 40\\n# SHUFFLE_BUFFER_SIZE = 10000\\npatience = 5\\n\\n\\ndef fixed_text_to_square_image(values):\\n    return text_to_square_image(values, image_size=image_size, cut_length=cut_length)\\n\\n\\nsteps_per_epoch = np.ceil(train_indices.shape[0] / BATCH_SIZE)\\nsteps_per_epoch_val = np.ceil(valid_indices.shape[0] / BATCH_SIZE)\\n\\ndataset = build_tf_dataset(\\n    X[train_indices],\\n    Y[train_indices],\\n    image_size,\\n    fixed_text_to_square_image,\\n    PREFETCH,\\n    BATCH_SIZE,\\n)\\n\\ndataset_valid = build_tf_dataset(\\n    X[valid_indices],\\n    Y[valid_indices],\\n    image_size,\\n    fixed_text_to_square_image,\\n    PREFETCH,\\n    BATCH_SIZE,\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "epochs_1 = 2\n",
    "epochs_2 = 200\n",
    "image_size = 96\n",
    "cut_length = None\n",
    "BATCH_SIZE = 256\n",
    "PREFETCH = 100  # 40\n",
    "# SHUFFLE_BUFFER_SIZE = 10000\n",
    "patience = 5\n",
    "\n",
    "\n",
    "def fixed_text_to_square_image(values):\n",
    "    return text_to_square_image(values, image_size=image_size, cut_length=cut_length)\n",
    "\n",
    "\n",
    "steps_per_epoch = np.ceil(train_indices.shape[0] / BATCH_SIZE)\n",
    "steps_per_epoch_val = np.ceil(valid_indices.shape[0] / BATCH_SIZE)\n",
    "\n",
    "dataset = build_tf_dataset(\n",
    "    X[train_indices],\n",
    "    Y[train_indices],\n",
    "    image_size,\n",
    "    fixed_text_to_square_image,\n",
    "    PREFETCH,\n",
    "    BATCH_SIZE,\n",
    ")\n",
    "\n",
    "dataset_valid = build_tf_dataset(\n",
    "    X[valid_indices],\n",
    "    Y[valid_indices],\n",
    "    image_size,\n",
    "    fixed_text_to_square_image,\n",
    "    PREFETCH,\n",
    "    BATCH_SIZE,\n",
    ")"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "for image, label in dataset.as_numpy_iterator():\n",
    "    # print(label)\n",
    "    imshow(image[0])\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 16;\n",
       "                var nbb_unformatted_code = \"activation = mish\\noptimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)\";\n",
       "                var nbb_formatted_code = \"activation = mish\\noptimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "activation = mish\n",
    "optimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 17;\n",
       "                var nbb_unformatted_code = \"from tensorflow.keras.applications.inception_v3 import InceptionV3\\nfrom tensorflow.keras.preprocessing import image\\nfrom tensorflow.keras.models import Model\\nfrom tensorflow.keras.layers import (\\n    Dense,\\n    GlobalAveragePooling2D,\\n    BatchNormalization,\\n    Dropout,\\n)\\nfrom tensorflow.keras import backend as K\\n\\n# create the base pre-trained model\\nbase_model = InceptionV3(weights=\\\"imagenet\\\", include_top=False)\\n\\n# add a global spatial average pooling layer\\nx = base_model.output\\nx = GlobalAveragePooling2D()(x)\\n# let's add a fully-connected layer\\nx = Dense(1024, activation=activation, kernel_initializer=\\\"he_normal\\\")(x)\\nx = Dropout(0.2)(x)\\nx = Dense(128, activation=activation, kernel_initializer=\\\"he_normal\\\")(x)\\nx = Dropout(0.2)(x)\\n\\n# and a logistic layer -- let's say we have 200 classes\\n# predictions = Dense(200, activation='softmax')(x)\\npredictions = Dense(2, activation=\\\"softmax\\\")(x)\\n\\n# this is the model we will train\\nmodel = Model(inputs=base_model.input, outputs=predictions)\\n\\n# first: train only the top layers (which were randomly initialized)\\n# i.e. freeze all convolutional InceptionV3 layers\\nfor layer in base_model.layers:\\n    layer.trainable = False\\n\\n# es.set_model(model)\\n# compile the model (should be done *after* setting layers to non-trainable)\\nmodel.compile(optimizer=optimizer, loss=\\\"binary_crossentropy\\\")  # , metrics=[\\\"AUC\\\"])\";\n",
       "                var nbb_formatted_code = \"from tensorflow.keras.applications.inception_v3 import InceptionV3\\nfrom tensorflow.keras.preprocessing import image\\nfrom tensorflow.keras.models import Model\\nfrom tensorflow.keras.layers import (\\n    Dense,\\n    GlobalAveragePooling2D,\\n    BatchNormalization,\\n    Dropout,\\n)\\nfrom tensorflow.keras import backend as K\\n\\n# create the base pre-trained model\\nbase_model = InceptionV3(weights=\\\"imagenet\\\", include_top=False)\\n\\n# add a global spatial average pooling layer\\nx = base_model.output\\nx = GlobalAveragePooling2D()(x)\\n# let's add a fully-connected layer\\nx = Dense(1024, activation=activation, kernel_initializer=\\\"he_normal\\\")(x)\\nx = Dropout(0.2)(x)\\nx = Dense(128, activation=activation, kernel_initializer=\\\"he_normal\\\")(x)\\nx = Dropout(0.2)(x)\\n\\n# and a logistic layer -- let's say we have 200 classes\\n# predictions = Dense(200, activation='softmax')(x)\\npredictions = Dense(2, activation=\\\"softmax\\\")(x)\\n\\n# this is the model we will train\\nmodel = Model(inputs=base_model.input, outputs=predictions)\\n\\n# first: train only the top layers (which were randomly initialized)\\n# i.e. freeze all convolutional InceptionV3 layers\\nfor layer in base_model.layers:\\n    layer.trainable = False\\n\\n# es.set_model(model)\\n# compile the model (should be done *after* setting layers to non-trainable)\\nmodel.compile(optimizer=optimizer, loss=\\\"binary_crossentropy\\\")  # , metrics=[\\\"AUC\\\"])\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tensorflow.keras.applications.inception_v3 import InceptionV3\n",
    "from tensorflow.keras.preprocessing import image\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import (\n",
    "    Dense,\n",
    "    GlobalAveragePooling2D,\n",
    "    BatchNormalization,\n",
    "    Dropout,\n",
    ")\n",
    "from tensorflow.keras import backend as K\n",
    "\n",
    "# create the base pre-trained model\n",
    "base_model = InceptionV3(weights=\"imagenet\", include_top=False)\n",
    "\n",
    "# add a global spatial average pooling layer\n",
    "x = base_model.output\n",
    "x = GlobalAveragePooling2D()(x)\n",
    "# let's add a fully-connected layer\n",
    "x = Dense(1024, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "x = Dropout(0.2)(x)\n",
    "x = Dense(128, activation=activation, kernel_initializer=\"he_normal\")(x)\n",
    "x = Dropout(0.2)(x)\n",
    "\n",
    "# and a logistic layer -- let's say we have 200 classes\n",
    "# predictions = Dense(200, activation='softmax')(x)\n",
    "predictions = Dense(2, activation=\"softmax\")(x)\n",
    "\n",
    "# this is the model we will train\n",
    "model = Model(inputs=base_model.input, outputs=predictions)\n",
    "\n",
    "# first: train only the top layers (which were randomly initialized)\n",
    "# i.e. freeze all convolutional InceptionV3 layers\n",
    "for layer in base_model.layers:\n",
    "    layer.trainable = False\n",
    "\n",
    "# es.set_model(model)\n",
    "# compile the model (should be done *after* setting layers to non-trainable)\n",
    "model.compile(optimizer=optimizer, loss=\"binary_crossentropy\")  # , metrics=[\"AUC\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train for 230.0 steps, validate for 29.0 steps\n",
      "Epoch 1/2\n",
      "WARNING:tensorflow:From /work/.cache/poetry/thc-net-KQLMmzPP-py3.7/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1786: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "230/230 [==============================] - 246s 1s/step - loss: 0.2582 - val_loss: 1.2114\n",
      "Epoch 2/2\n",
      "230/230 [==============================] - 22s 95ms/step - loss: 0.1962 - val_loss: 0.9448\n",
      "CPU times: user 1min 46s, sys: 18.1 s, total: 2min 4s\n",
      "Wall time: 4min 28s\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 18;\n",
       "                var nbb_unformatted_code = \"%%time\\n# train the model on the new data for a few epochs\\nhistory_1 = model.fit(\\n    dataset, \\n    #callbacks=[es],\\n    epochs=epochs_1,\\n    steps_per_epoch=steps_per_epoch,\\n    validation_data=dataset_valid,\\n    validation_steps=steps_per_epoch_val\\n)\\n# at this point, the top layers are well trained and we can start fine-tuning\\n# convolutional layers from inception V3. We will freeze the bottom N layers\\n# and train the remaining top layers.\";\n",
       "                var nbb_formatted_code = \"%%time\\n# train the model on the new data for a few epochs\\nhistory_1 = model.fit(\\n    dataset, \\n    #callbacks=[es],\\n    epochs=epochs_1,\\n    steps_per_epoch=steps_per_epoch,\\n    validation_data=dataset_valid,\\n    validation_steps=steps_per_epoch_val\\n)\\n# at this point, the top layers are well trained and we can start fine-tuning\\n# convolutional layers from inception V3. We will freeze the bottom N layers\\n# and train the remaining top layers.\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "# train the model on the new data for a few epochs\n",
    "history_1 = model.fit(\n",
    "    dataset, \n",
    "    #callbacks=[es],\n",
    "    epochs=epochs_1,\n",
    "    steps_per_epoch=steps_per_epoch,\n",
    "    validation_data=dataset_valid,\n",
    "    validation_steps=steps_per_epoch_val\n",
    ")\n",
    "# at this point, the top layers are well trained and we can start fine-tuning\n",
    "# convolutional layers from inception V3. We will freeze the bottom N layers\n",
    "# and train the remaining top layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 19;\n",
       "                var nbb_unformatted_code = \"def plot_metric(history, metric):\\n    # Plot training & validation loss values\\n    plt.plot(history.history[metric])\\n    plt.plot(history.history[f\\\"val_{metric}\\\"])\\n    plt.title(f\\\"Model {metric}\\\")\\n    plt.ylabel(f\\\"{metric}\\\")\\n    plt.xlabel(\\\"Epoch\\\")\\n    plt.legend([\\\"Train\\\", \\\"Test\\\"], loc=\\\"upper left\\\")\\n    plt.show()\";\n",
       "                var nbb_formatted_code = \"def plot_metric(history, metric):\\n    # Plot training & validation loss values\\n    plt.plot(history.history[metric])\\n    plt.plot(history.history[f\\\"val_{metric}\\\"])\\n    plt.title(f\\\"Model {metric}\\\")\\n    plt.ylabel(f\\\"{metric}\\\")\\n    plt.xlabel(\\\"Epoch\\\")\\n    plt.legend([\\\"Train\\\", \\\"Test\\\"], loc=\\\"upper left\\\")\\n    plt.show()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_metric(history, metric):\n",
    "    # Plot training & validation loss values\n",
    "    plt.plot(history.history[metric])\n",
    "    plt.plot(history.history[f\"val_{metric}\"])\n",
    "    plt.title(f\"Model {metric}\")\n",
    "    plt.ylabel(f\"{metric}\")\n",
    "    plt.xlabel(\"Epoch\")\n",
    "    plt.legend([\"Train\", \"Test\"], loc=\"upper left\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de3xcdZ3/8ddnZjJJ2qa3JIVC2qZAW1ou3iKIyAIiPLgJPH6yCisKLspDfg+5/FB/4rI/YV3dxetvRVmxKrKAcl+lCi78dKmogNICcmm51HJpuDUNJW1J0tw+vz/OSedkZtJO2pyZZs77+XicR2bO+c7M9zRp3vlezveYuyMiIsmVqnQFRESkshQEIiIJpyAQEUk4BYGISMIpCEREEk5BICKScAoCkRKYWauZuZllSih7jpn9YVffR6RcFARSdczsBTPrM7OmvP2Phr+EWytTM5Hdk4JAqtXzwJnDT8zsIGBS5aojsvtSEEi1ugH4eOT52cD10QJmNs3MrjezDjN70cz+0cxS4bG0mX3TzDaY2VrgpCKv/bGZvWpmL5vZV8wsPdZKmtleZrbMzN4wszVm9qnIsUPMbIWZbTKz183s2+H+OjO70cw6zexNM3vYzPYY62eLDFMQSLV6CJhqZovDX9BnADfmlfkuMA3YBziSIDg+ER77FHAy8A6gDTg977XXAQPAfmGZ44BP7kQ9bwbagb3Cz/gXM3t/eOw7wHfcfSqwL3BruP/ssN5zgEbg00DPTny2CKAgkOo23Co4FlgNvDx8IBIOX3T3ze7+AvAt4GNhkQ8D/+bu69z9DeBfI6/dAzgRuNjd33L39cD/Dd+vZGY2Bzgc+IK797r7Y8CPyLVk+oH9zKzJ3be4+0OR/Y3Afu4+6O4r3X3TWD5bJEpBINXsBuDvgHPI6xYCmoAa4MXIvheBvcPHewHr8o4Nmxe+9tWwa+ZN4AfArDHWby/gDXffPEodzgUWAk+H3T8nR87rHuBmM3vFzL5uZjVj/GyRbRQEUrXc/UWCQeMTgf/MO7yB4C/reZF9c8m1Gl4l6HqJHhu2DtgKNLn79HCb6u4HjLGKrwAzzayhWB3c/Tl3P5MgYL4G3G5mk929393/yd2XAO8l6ML6OCI7SUEg1e5c4P3u/lZ0p7sPEvS5f9XMGsxsHnAJuXGEW4ELzazFzGYAl0Ze+ypwL/AtM5tqZikz29fMjhxLxdx9HfAA8K/hAPDBYX1vBDCzs8ys2d2HgDfDlw2Z2dFmdlDYvbWJINCGxvLZIlEKAqlq7v5Xd18xyuELgLeAtcAfgJ8B14bHfkjQ/fIX4BEKWxQfB7LAKmAjcDsweyeqeCbQStA6+Dlwubv/Jjx2PPCUmW0hGDg+w917gD3Dz9tEMPbxO4LuIpGdYroxjYhIsqlFICKScAoCEZGEUxCIiCScgkBEJOEm3FK4TU1N3traWulqiIhMKCtXrtzg7s3Fjk24IGhtbWXFitFmA4qISDFm9uJox9Q1JCKScAoCEZGEUxCIiCTchBsjKKa/v5/29nZ6e3srXZXY1dXV0dLSQk2NFpsUkfFRFUHQ3t5OQ0MDra2tmFmlqxMbd6ezs5P29nbmz59f6eqISJWoiq6h3t5eGhsbqzoEAMyMxsbGRLR8RKR8qiIIgKoPgWFJOU8RKZ/YgsDMrjWz9Wb25CjHP2pmj5vZE2b2gJm9La66ANDfC5tehZ6N0N8DruXbRUQg3jGC64DvUXiLwGHPA0e6+0YzOwFYChwaW20GemDLa5EdBplayNRDTR1k6oKv6VoY41/dnZ2dHHPMMQC89tprpNNpmpuDC/j+/Oc/k81mR33tihUruP7667nqqqvGfEoiIuMhtiBw9/vNrHU7xx+IPH0IaImrLgDUz4DaaTDQm9v6e6D/LejdGCmYgpq8gMjUQ7pm1IBobGzkscceA+CKK65gypQpfO5zn9t2fGBggEym+D91W1sbbW1t43aaIiJjtbvMGjoX+PVoB83sPOA8gLlz545WbMdSKchOCraoocFIOPQGrYetm6HnjUglUmGroT4Mh/BxKlM0IM455xzq6up49NFHOfzwwznjjDO46KKL6O3tpb6+np/85CcsWrSI5cuX881vfpNf/epXXHHFFbz00kusXbuWl156iYsvvpgLL7xw589XRKQEFQ8CMzuaIAjeN1oZd19K0HVEW1vbdm+p9k+/fIpVr2wap9o5+BBLZtVz+TFNQUj0dsFQZ66IpXPhUFMHA30wFIw/tLe388ADD5BOp9m0aRO///3vyWQy/OY3v+Ef/uEfuOOOOwo+8emnn+a+++5j8+bNLFq0iPPPP1/XDIhIrCoaBOHNun8EnODunTsqX34W/qKvg2lzcrsH+0e2Hvp7g0Ho7kHo6YRUD/Rs5G+P/xvSW16FTB1dHZ2c/dkv8NyaNZgZ/f39RT/xpJNOora2ltraWmbNmsXrr79OS0u8vWYikmwVCwIzm0twQ/CPufuz4/W+l3/wgPF6q9Gla4KttiG3zz0IiPqZUJuFdA2TJ9VCdyf4EP/ni5dz9Lv25+c/+CovvLKBo047C7rfCEIkct/o2tra3Mek0wwMDMR/PiKSaLEFgZndBBwFNJlZO3A5UAPg7tcAXwIagX8P58YPuPvEHTU1g0w2mIlUOwWyU2Dq3rDnwTDYR1fvEHvPXwjZyVz3s+8H4xJvvghd62DrJnh9VdCqyAyFX+sqfUYikhBxzho6cwfHPwl8Mq7P321YME31f3/xMs4++2y+8q2rOemkkyCdhebF0PBybuB5aAD6NsPGF4LXDvTChjUw1UcOUvt2h0lERMbEfIL9Umlra/P8G9OsXr2axYsXV6hG48yHYGBrMLU1Og4x2LetyOqXOlj82Jdh1hKYtTgIlFmLg3GMVNVcLC4i48jMVo7W61LxWUOSx1LBX/819SP3R6e4vtYLU/aAF/4Ij9+SK5OdAs2LglCYtQSa9w++Nuw55ovkRCQ5FAQTRSoN2cnBVj8dzgqnnvZ2wfqnoWM1rF8N61fBs/fCozfmXls3PQyHvICY3FiZcxGR3YqCYKKrmwZzDw22qLc2BMHQ8XQQDutXw5N3QO+1uTKTZ8Gs/fO6mPYP3lNEEkNBUK0mN8H8I4JtmDtsfi0IhmhAPHJDsNTGsKktYUBEWhDNi4LWiIhUHQVBkpjB1NnBtt8xuf1DQ8E01vWrR3YxPf97GNw6/GKYMa9wgLppQTBlVkQmLAWBBDONZswLtkXH5/YPDcIbzxe2IJ67N5jqCsGV1437Fo4/zNwH0vrxEpkI9D91HOzKMtQAy5cvJ5vN8t73vjf2uo5JKg1N+wUbp+T2D/RB55pIQKyG156EVcuAcDpyOgtNC3OD1MMtiOnzNMVVZDejIBgHO1qGekeWL1/OlClTdr8gGE0mC3ssCbaovm7Y8OzI1sNLf4InbsuVqZkUTnHN62KaupemuIpUiIIgJitXruSSSy5hy5YtNDU1cd111zF79myuuuoqrrnmGjKZDEuWLOHKK6/kmmuuIZ1Oc+ONN/Ld736XI444YscfsDvKToK93h5sUb2boOOZkeMPa34Lj/00V6Z2WuEA9awlMKW5vOcgkkDVFwS/vhRee2J833PPg+CEK0su7u5ccMEF3HnnnTQ3N3PLLbdw2WWXce2113LllVfy/PPPU1tby5tvvsn06dP59Kc/PeZWxIRSNxXmvDvYorrfyBugXg2r7oSV1+XKTGqKXAMRmeJaP6OspyBSzaovCHYDW7du5cknn+TYY48FYHBwkNmzZwNw8MEH89GPfpTTTjuN0047rZLVrLxJM6H18GAb5g5b1hcOUD92U7AO07CGvQqvgWheFCz4JyJjUn1BMIa/3OPi7hxwwAE8+OCDBcfuuusu7r//fn75y1/y1a9+lSeeGOfWy0RnBg17BNu+R+f2u0NXe+EU14d/HKzFNGz6vMKrqJsWBveUEJGiqi8IdgO1tbV0dHTw4IMPcthhh9Hf38+zzz7L4sWLWbduHUcffTTve9/7uPnmm9myZQsNDQ1s2jRed1WrUmYwfU6wLTwut39oMFitNb+Lac1vYSi8+Y+lYOa+hS2Ixn2D+0qIJJyCIAapVIrbb7+dCy+8kK6uLgYGBrj44otZuHAhZ511Fl1dXbg7F154IdOnT+eDH/wgp59+OnfeeefEHiyuhFR4HUPjvrD45Nz+wX7o/GteF9PT8PRdwQqvAKmacIpr3iD1jNbgfUUSQstQT0BJO99x1d9bOMV1/ergJkHDMvWRVVwjATGtRVNcZcLSMtQiw2rqYPbBwRa1dUvhFNe1v4O/3JQrk20YZYrrLAWETGgKAhEIZhu1vCvYono25i3zvTroXnrk+lyZ+plFprguDmZFiUwAVRME7o4l4K+yidaVN+HVz4B5hwVb1JaOwimuj98GW7tyZabsWXyZ79qG8p6DyA5URRDU1dXR2dlJY2NjVYeBu9PZ2UldnaZCVtyUZphyJOxzZG6fO2x6pXCK68rroL87V27a3OLLfOfflU6kTKoiCFpaWmhvb6ejo6PSVYldXV0dLS0tla6GFGMG0/YOtgUfyO0fGgoGo/OnuK5dnrsXtaVgxvzCLqbG/YK1nURiVBVBUFNTw/z58ytdDZHiUimYOT/Y9j8xt39wAN5YW9jF9MyvwQfD12agcUGui2nbMt/zNcVVxk1VBIHIhJTOQPPCYIsa2AobnhvZgnjlUXjqF2xb5jtTF9wUKH8V12lztMy3jJmCQGR3k6mFPQ8Mtqi+t4IprtGAeOEP8PgtuTLZKZFrICItiIY9NcVVRqUgEJkospNh73cGW1RvV94U11Xw7L3w6I25MnXTCy+Qm7UEJjeW9xxkt6QgEJno6qbB3EODLeqtDWHrITL+8OQd0HttrszkWcWnuNZNK+85SEUpCESq1eQmmH9EsA1zh82vFQ5QP3ID9L+VKze1pfgU1+zk8p+HxE5BIJIkZjB1drDtd0xu/9AQdK0rvAbi+d/D4NbhFwcL8hUs870gGNeQCUtBICLBTKMZ84Jt0fG5/YMD4TLfeS2I5+6FoYGgjKWD6x3yu5hm7hPMjJLdnr5LIjK6dAaa9gs2TsntH+iDzjWRgFgd3CJ21TK2TXFNZ6FpUWEX0/R5muK6m4ktCMzsWuBkYL27H1jkuAHfAU4EuoFz3P2RuOojIuMok4U9lgRbVF93sMx3tIvppYfgidtyZWomhbOW8hbpm7qXprhWSJwtguuA7wHXj3L8BGBBuB0KfD/8KiITVXYS7PX2YIvq3RReAxHpYlrzG3jsp7kytdNGWea7ubznkECxBYG7329mrdspcipwvQfLaT5kZtPNbLa7vxpXnUSkQuqmwpx3B1tU9xu5genhLqZVdwYL9Q2b1FRkme/9g5VhZVxUcoxgb2Bd5Hl7uK8gCMzsPOA8gLlz55alciJSBpNmQuvhwTbMHbaszw1MD3cxPXYT9G3OlWvYq3CAunlRcG8JGZMJMVjs7kuBpRDcqrLC1RGROJlBwx7Btu/Ruf3u0NVeOMX14R/DQE+u3PR5Raa4LgzuTidFVTIIXgbmRJ63hPtERAqZwfQ5wbbwuNz+ocFwimveMt9rfgtD/eFrUzBz38IWROO+kK6pyOnsTioZBMuAz5jZzQSDxF0aHxCRMUulg1/ojfvC4pNz+wf7ofOvhddAPH0X+FD42pqgtZA/SD2jNVHLfMc5ffQm4CigyczagcuBGgB3vwa4m2Dq6BqC6aOfiKsuIpJA6ZrwF/z+I/f39xZOcW1/OFiHaVimPrKKayQgprVU5RRXm2j3wG1ra/MVK1ZUuhoiUm22bimc4rp+NWyOdFRkG0aZ4jprtw8IM1vp7m3Fjk2IwWIRkdjVToGWdwVbVM/GYJnv6BTXp++CRyKXSNXPLDLFdXEwK2oCUBCIiGxP/QyYd1iwRW3pKJzi+vhtsLUrV2bKnsWX+a5tKO857ICCQERkZ0xphilHwj5H5va5w6ZXCqe4rrwO+rtz5abNLb7Md0192U8DFAQiIuPHDKbtHWwLPpDbPzQEb75YOMV17XIY7Atfm4IZ8wu7mBr3C9Z2ipGCQEQkbqkUzJwfbPufmNs/OABvrC0coH7m1+CD4Wsz0LggaEEc8D9gySnFP2MXKAhERColnYHmhcEWNbAVNjw3sgXxyqOw50GxVENBICKyu8nUwp4HBltUTNP9dXcIEZGJIqZrFRQEIiIJpyAQEUk4BYGISMIpCEREEk5BICKScAoCEZGEUxCIiCScgkBEJOEUBCIiCacgEBFJOAWBiEjCKQhERBJOQSAiknAKAhGRhFMQiIgknIJARCThFAQiIgmnIBARSTgFgYhIwikIREQSTkEgIpJwCgIRkYSLNQjM7Hgze8bM1pjZpUWOzzWz+8zsUTN73MxOjLM+IiJSKLYgMLM0cDVwArAEONPMluQV+0fgVnd/B3AG8O9x1UdERIqLs0VwCLDG3de6ex9wM3BqXhkHpoaPpwGvxFgfEREpIs4g2BtYF3neHu6LugI4y8zagbuBC4q9kZmdZ2YrzGxFR0dHHHUVEUmsSg8Wnwlc5+4twInADWZWUCd3X+rube7e1tzcXPZKiohUsziD4GVgTuR5S7gv6lzgVgB3fxCoA5pirJOIiOSJMwgeBhaY2XwzyxIMBi/LK/MScAyAmS0mCAL1/YiIlFFsQeDuA8BngHuA1QSzg54ysy+b2Slhsc8CnzKzvwA3Aee4u8dVJxERKZSJ883d/W6CQeDovi9FHq8CDo+zDiIisn2VHiwWEZEKUxCIiCScgkBEJOEUBCIiCacgEBFJOAWBiEjCKQhERBKupCAws4vMbKoFfmxmj5jZcXFXTkRE4ldqi+Dv3X0TcBwwA/gYcGVstRIRkbIpNQgs/HoicIO7PxXZJyIiE1ipQbDSzO4lCIJ7zKwBGIqvWiIiUi6lrjV0LvB2YK27d5vZTOAT8VVLRETKpdQWwWHAM+7+ppmdRXCv4a74qiUiIuVSahB8H+g2s7cRLB39V+D62GolIiJlU2oQDIT3CTgV+J67Xw00xFctEREpl1LHCDab2RcJpo0eEd5XuCa+aomISLmU2iL4CLCV4HqC1wjuP/yN2GolIiJlU1IQhL/8fwpMM7OTgV531xiBiEgVKHWJiQ8Dfwb+Fvgw8CczOz3OiomISHmUOkZwGfBud18PYGbNwG+A2+OqmIiIlEepYwSp4RAIdY7htSIishsrtUXwX2Z2D3BT+PwjwN3xVElERMqppCBw98+b2YeAw8NdS9395/FVS0REyqXUFgHufgdwR4x1ERGRCthuEJjZZsCLHQLc3afGUisRESmb7QaBu2sZCRGRKqeZPyIiCacgEBFJOAWBiEjCKQhERBIu1iAws+PN7BkzW2Nml45S5sNmtsrMnjKzn8VZHxERKVTydQRjZWZp4GrgWKAdeNjMlrn7qkiZBcAXgcPdfaOZzYqrPiIiUlycLYJDgDXuvtbd+4CbCe5wFvUp4Gp33wiQt56RiIiUQZxBsDewLvK8PdwXtRBYaGZ/NLOHzOz4Ym9kZueZ2QozW9HR0RFTdUVEkqnSg8UZYAFwFHAm8EMzm55fyN2Xunubu7c1NzeXuYoiItUtziB4GZgTed4S7otqB5a5e7+7Pw88SxAMIiJSJnEGwcPAAjObb2ZZ4AxgWV6ZXxC0BjCzJoKuorUx1klERPLEFgTuPgB8BrgHWA3c6u5PmdmXzeyUsNg9QKeZrQLuAz7v7p1x1UlERAqZe7HFRXdfbW1tvmLFikpXQ0RkQjGzle7eVuxYpQeLRUSkwhQEIiIJpyAQEUk4BYGISMIpCEREEk5BICKScAoCEZGEUxCIiCScgkBEJOEUBCIiCacgEBFJOAWBiEjCKQhERBJOQSAiknAKAhGRhFMQiIgknIJARCThFAQiIgmnIBARSTgFgYhIwikIREQSTkEgIpJwCgIRkYRTEIiIJJyCQEQk4RQEIiIJpyAQEUk4BYGISMIpCEREEk5BICKScLEGgZkdb2bPmNkaM7t0O+U+ZGZuZm1x1kdERArFFgRmlgauBk4AlgBnmtmSIuUagIuAP8VVFxERGV2cLYJDgDXuvtbd+4CbgVOLlPtn4GtAb4x1ERGRUcQZBHsD6yLP28N925jZO4E57n7X9t7IzM4zsxVmtqKjo2P8ayoikmAVGyw2sxTwbeCzOyrr7kvdvc3d25qbm+OvnIhIgsQZBC8DcyLPW8J9wxqAA4HlZvYC8B5gmQaMRUTKK84geBhYYGbzzSwLnAEsGz7o7l3u3uTure7eCjwEnOLuK2Ksk4iI5IktCNx9APgMcA+wGrjV3Z8ysy+b2Slxfa6IiIxNJs43d/e7gbvz9n1plLJHxVkXEREpTlcWi4gknIJARCThFAQiIgmnIBARSTgFgYhIwikIREQSTkEgIpJwCgIRkYRTEIiIJJyCQEQk4RQEIiIJpyAQEUk4BYGISMIpCEREEk5BICKScAoCEZGEUxCIiCScgkBEJOEUBCIiCacgEBFJOAWBiEjCZSpdgXL57erX+eJ/PkF9Nk19TZq6muDr8PPo123HalLB/mwmfJ6mPpvadnxSuL8umyKbTmFmlT5NEZExS0wQNDfUcsziWfT0DdLdN0hP/yC9/YOs39xPT98gvf1DdPcNhPuHxvz+KWNEkEyKBs5wyNSkqcummZQfOEXCKP+1k7JpajMKGxEZf4kJgoNbpnNwy/SSyg4NOVsHhujpDwKjpy/cIs97o8f6Rx7vjTzu7hvkjbf6th3vDct29w/iPvbzGBkUQYtlUk2GumzYgskLmUnZ7YdRfuDU1aRJpxQ2IkmSmCAYi1TKwi6hdGyf4R6ETX6g9IbhkR8cPf1hMIWtlp6+ka/d1NPP610jw6i7f5DBobGnTW0mNTI4irVaClo1qZK60IaPZ9IanhLZXSgIKsTMqAt/yZbWTtk5/YNDdPdFA2Vkq6W7IHAiQRI91j9Id98AnW/1hWE1sK1LrW9w7F1pNWkruQut2PGiYzrZSAuoJk1N2tSVJlICBUGVq0mnmFafYlp9TWyfMTA4RO/A0A670Lrzus2ix6Nh1dXTP6JV09M3yNaBsYdNOmWREEntsAttOIDyWzyjhVW9xm2kSigIZJdl0immpFNMqY3vx2loyOkd2EEXWthllt+FVtDi6Rtkw5a+wrDaiXEbG54kMIYutEnZzMgutZr0yG60SLdafTZNXSZNSuM2EiMFgUwIqZQxKZthUja+H9nhcZtirZpSutDyj7/Z08+rXT0jxnS6+wbYiWGbYGJATfFWy2gzzUqZiRZ9rkkCyaUgEAlFx21mxPQZ7k7/oBeETf6Msu11oUXHeLZsHaBj89aCsOofHHvaZNMp6mpSYatkuJWSGhEipXShFQ2r7PC4jSYJ7I5iDQIzOx74DpAGfuTuV+YdvwT4JDAAdAB/7+4vxlknkUoyM7IZI5tJMY34xm36BwtnpI0MnB13oXVHAmjjW/0jutC6+wbp24lxm8zwuE1+q6Sg1ZLXhRYG0va60IZfq3GbsYstCMwsDVwNHAu0Aw+b2TJ3XxUp9ijQ5u7dZnY+8HXgI3HVSSQpatIpatIpGuriC5vBIR8RNtFWSf71NPkTCUZ2sQ3R0zcw4uLOaHCNVf7FnUUv2hxDF1uxa3FqM6mqGreJs0VwCLDG3dcCmNnNwKnAtiBw9/si5R8CzoqxPiIyjtIpY3JthskxThJw91wwFOsmK3Jx54husrww2vhWHy/HeHFnfhfayCVrRnaTlRJG5Rq3iTMI9gbWRZ63A4dup/y5wK+LHTCz84DzAObOnTte9ROR3ZxZeS7u7BscordviO7+gaJdaN19AwUXd24Lkrww2twbjNts617bhYs7s5nUiOnNf3foXD55xD7j/m+wWwwWm9lZQBtwZLHj7r4UWArQ1ta2E9ktIlKcmVGbSVObScc+blN0plmRiQPFl6wZormhNpa6xRkELwNzIs9bwn0jmNkHgMuAI919a4z1ERGpmOFxm6kxjtvsrDjncj0MLDCz+WaWBc4AlkULmNk7gB8Ap7j7+hjrIiIio4gtCNx9APgMcA+wGrjV3Z8ysy+b2SlhsW8AU4DbzOwxM1s2ytuJiEhMYh0jcPe7gbvz9n0p8vgDcX6+iIjsmC7zExFJOAWBiEjCKQhERBJOQSAiknAKAhGRhDPfmUU2KsjMOoCdXaG0CdgwjtWZCHTOyaBzToZdOed57t5c7MCEC4JdYWYr3L2t0vUoJ51zMuickyGuc1bXkIhIwikIREQSLmlBsLTSFagAnXMy6JyTIZZzTtQYgYiIFEpai0BERPIoCEREEq4qg8DMjjezZ8xsjZldWuR4rZndEh7/k5m1lr+W46uEc77EzFaZ2eNm9lszm1eJeo6nHZ1zpNyHzMzNbMJPNSzlnM3sw+H3+ikz+1m56zjeSvjZnmtm95nZo+HP94mVqOd4MbNrzWy9mT05ynEzs6vCf4/Hzeydu/yh7l5VG5AG/grsA2SBvwBL8sr8T+Ca8PEZwC2VrncZzvloYFL4+PwknHNYrgG4H3gIaKt0vcvwfV4APArMCJ/PqnS9y3DOS4Hzw8dLgBcqXe9dPOe/Ad4JPDnK8RMJ7u9uwHuAP+3qZ1Zji+AQYI27r3X3PuBm4NS8MqcC/xE+vh04xsysjHUcbzs8Z3e/z927w6cPEdw6dCIr5fsM8M/A14DeclYuJqWc86eAq919I4BP/Dv/lXLODkwNH08DXilj/cadu98PvLGdIqcC13vgIWC6mc3elc+sxiDYG1gXed4e7itaxoM7qXUBjWWpXTxKOeeocwn+opjIdnjOYZN5jrvfVc6KxaiU7/NCYKGZ/dHMHjKz48tWu3iUcs5XAGeZWTvBjbAuKE/VKmas/993KNY7lMnux8zOAtqAIytdlziZWQr4NnBOhatSbhmC7qGjCFp995vZQe7+ZkVrFa8zgevc/Vtmdhhwg5kd6O5Dla7YRFGNLYKXgTmR5y3hvqJlzCxD0JzsLEvt4lHKOWNmHwAuA05x961lqltcdnTODcCBwHIze4GgL3XZBB8wLuX73A4sc/d+d38eeJYgGCaqUs75XOBWAHd/EKgjWJytWpX0/30sqrtmeq0AAAKpSURBVDEIHgYWmNl8M8sSDAYvyyuzDDg7fHw68N8ejsJMUDs8ZzN7B/ADghCY6P3GsINzdvcud29y91Z3byUYFznF3VdUprrjopSf7V8QtAYwsyaCrqK15azkOCvlnF8CjgEws8UEQdBR1lqW1zLg4+HsofcAXe7+6q68YdV1Dbn7gJl9BriHYMbBte7+lJl9GVjh7suAHxM0H9cQDMqcUbka77oSz/kbwBTgtnBc/CV3P6Vild5FJZ5zVSnxnO8BjjOzVcAg8Hl3n7Ct3RLP+bPAD83sfxEMHJ8zkf+wM7ObCMK8KRz3uByoAXD3awjGQU4E1gDdwCd2+TMn8L+XiIiMg2rsGhIRkTFQEIiIJJyCQEQk4RQEIiIJpyAQEUk4BYFIHjMbNLPHItuoK5vuxHu3jraqpEilVN11BCLjoMfd317pSoiUi1oEIiUysxfM7Otm9oSZ/dnM9gv3t5rZf0fu9TA33L+Hmf3czP4Sbu8N3yptZj8M7xdwr5nVV+ykRFAQiBRTn9c19JHIsS53Pwj4HvBv4b7vAv/h7gcDPwWuCvdfBfzO3d9GsL78U+H+BQRLRR8AvAl8KObzEdkuXVksksfMtrj7lCL7XwDe7+5rzawGeM3dG81sAzDb3fvD/a+6e5OZdQAt0QX+LLgb3v9z9wXh8y8ANe7+lfjPTKQ4tQhExsZHeTwW0ZVfB9FYnVSYgkBkbD4S+fpg+PgBcgsXfhT4ffj4twS3BcXM0mY2rVyVFBkL/SUiUqjezB6LPP8vdx+eQjrDzB4n+Kv+zHDfBcBPzOzzBMsfD68GeRGw1MzOJfjL/3xgl5YLFomDxghEShSOEbS5+4ZK10VkPKlrSEQk4dQiEBFJOLUIREQSTkEgIpJwCgIRkYRTEIiIJJyCQEQk4f4/kyuczY+Ttt4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 20;\n",
       "                var nbb_unformatted_code = \"plot_metric(history_1, \\\"loss\\\")\";\n",
       "                var nbb_formatted_code = \"plot_metric(history_1, \\\"loss\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_metric(history_1, \"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 21;\n",
       "                var nbb_unformatted_code = \"# plot_metric(history_1, \\\"AUC\\\")\";\n",
       "                var nbb_formatted_code = \"# plot_metric(history_1, \\\"AUC\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot_metric(history_1, \"AUC\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, None, None,  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, None, None, 3 864         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, None, None, 3 96          conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation (Activation)         (None, None, None, 3 0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, None, None, 3 9216        activation[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, None, None, 3 96          conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, None, None, 3 0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, None, None, 6 18432       activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, None, None, 6 192         conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, None, None, 6 0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D)    (None, None, None, 6 0           activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, None, None, 8 5120        max_pooling2d[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, None, None, 8 240         conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, None, None, 8 0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, None, None, 1 138240      activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, None, None, 1 576         conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, None, None, 1 0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, None, None, 1 0           activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, None, None, 6 12288       max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, None, None, 6 192         conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, None, None, 6 0           batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, None, None, 4 9216        max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, None, None, 9 55296       activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, None, None, 4 144         conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, None, None, 9 288         conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, None, None, 4 0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, None, None, 9 0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d (AveragePooli (None, None, None, 1 0           max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, None, None, 6 12288       max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, None, None, 6 76800       activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, None, None, 9 82944       activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, None, None, 3 6144        average_pooling2d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, None, None, 6 192         conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, None, None, 6 192         conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, None, None, 9 288         conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, None, None, 3 96          conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, None, None, 6 0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, None, None, 6 0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, None, None, 9 0           batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, None, None, 3 0           batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "mixed0 (Concatenate)            (None, None, None, 2 0           activation_5[0][0]               \n",
      "                                                                 activation_7[0][0]               \n",
      "                                                                 activation_10[0][0]              \n",
      "                                                                 activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, None, None, 6 16384       mixed0[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, None, None, 6 192         conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, None, None, 6 0           batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, None, None, 4 12288       mixed0[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, None, None, 9 55296       activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, None, None, 4 144         conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, None, None, 9 288         conv2d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, None, None, 4 0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, None, None, 9 0           batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_1 (AveragePoo (None, None, None, 2 0           mixed0[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, None, None, 6 16384       mixed0[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, None, None, 6 76800       activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, None, None, 9 82944       activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, None, None, 6 16384       average_pooling2d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, None, None, 6 192         conv2d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, None, None, 6 192         conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, None, None, 9 288         conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, None, None, 6 192         conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, None, None, 6 0           batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, None, None, 6 0           batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, None, None, 9 0           batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, None, None, 6 0           batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "mixed1 (Concatenate)            (None, None, None, 2 0           activation_12[0][0]              \n",
      "                                                                 activation_14[0][0]              \n",
      "                                                                 activation_17[0][0]              \n",
      "                                                                 activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)              (None, None, None, 6 18432       mixed1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNo (None, None, None, 6 192         conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, None, None, 6 0           batch_normalization_22[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, None, None, 4 13824       mixed1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_23 (Conv2D)              (None, None, None, 9 55296       activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, None, None, 4 144         conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNo (None, None, None, 9 288         conv2d_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, None, None, 4 0           batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_23 (Activation)      (None, None, None, 9 0           batch_normalization_23[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_2 (AveragePoo (None, None, None, 2 0           mixed1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, None, None, 6 18432       mixed1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)              (None, None, None, 6 76800       activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_24 (Conv2D)              (None, None, None, 9 82944       activation_23[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_25 (Conv2D)              (None, None, None, 6 18432       average_pooling2d_2[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, None, None, 6 192         conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNo (None, None, None, 6 192         conv2d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNo (None, None, None, 9 288         conv2d_24[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_25 (BatchNo (None, None, None, 6 192         conv2d_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, None, None, 6 0           batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, None, None, 6 0           batch_normalization_21[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_24 (Activation)      (None, None, None, 9 0           batch_normalization_24[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_25 (Activation)      (None, None, None, 6 0           batch_normalization_25[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "mixed2 (Concatenate)            (None, None, None, 2 0           activation_19[0][0]              \n",
      "                                                                 activation_21[0][0]              \n",
      "                                                                 activation_24[0][0]              \n",
      "                                                                 activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_27 (Conv2D)              (None, None, None, 6 18432       mixed2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_27 (BatchNo (None, None, None, 6 192         conv2d_27[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_27 (Activation)      (None, None, None, 6 0           batch_normalization_27[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_28 (Conv2D)              (None, None, None, 9 55296       activation_27[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_28 (BatchNo (None, None, None, 9 288         conv2d_28[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_28 (Activation)      (None, None, None, 9 0           batch_normalization_28[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_26 (Conv2D)              (None, None, None, 3 995328      mixed2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_29 (Conv2D)              (None, None, None, 9 82944       activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_26 (BatchNo (None, None, None, 3 1152        conv2d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_29 (BatchNo (None, None, None, 9 288         conv2d_29[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_26 (Activation)      (None, None, None, 3 0           batch_normalization_26[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_29 (Activation)      (None, None, None, 9 0           batch_normalization_29[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)  (None, None, None, 2 0           mixed2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "mixed3 (Concatenate)            (None, None, None, 7 0           activation_26[0][0]              \n",
      "                                                                 activation_29[0][0]              \n",
      "                                                                 max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_34 (Conv2D)              (None, None, None, 1 98304       mixed3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_34 (BatchNo (None, None, None, 1 384         conv2d_34[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_34 (Activation)      (None, None, None, 1 0           batch_normalization_34[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_35 (Conv2D)              (None, None, None, 1 114688      activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_35 (BatchNo (None, None, None, 1 384         conv2d_35[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_35 (Activation)      (None, None, None, 1 0           batch_normalization_35[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_31 (Conv2D)              (None, None, None, 1 98304       mixed3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_36 (Conv2D)              (None, None, None, 1 114688      activation_35[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_31 (BatchNo (None, None, None, 1 384         conv2d_31[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_36 (BatchNo (None, None, None, 1 384         conv2d_36[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_31 (Activation)      (None, None, None, 1 0           batch_normalization_31[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_36 (Activation)      (None, None, None, 1 0           batch_normalization_36[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_32 (Conv2D)              (None, None, None, 1 114688      activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_37 (Conv2D)              (None, None, None, 1 114688      activation_36[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_32 (BatchNo (None, None, None, 1 384         conv2d_32[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_37 (BatchNo (None, None, None, 1 384         conv2d_37[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_32 (Activation)      (None, None, None, 1 0           batch_normalization_32[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_37 (Activation)      (None, None, None, 1 0           batch_normalization_37[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_3 (AveragePoo (None, None, None, 7 0           mixed3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_30 (Conv2D)              (None, None, None, 1 147456      mixed3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_33 (Conv2D)              (None, None, None, 1 172032      activation_32[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_38 (Conv2D)              (None, None, None, 1 172032      activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_39 (Conv2D)              (None, None, None, 1 147456      average_pooling2d_3[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_30 (BatchNo (None, None, None, 1 576         conv2d_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_33 (BatchNo (None, None, None, 1 576         conv2d_33[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_38 (BatchNo (None, None, None, 1 576         conv2d_38[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_39 (BatchNo (None, None, None, 1 576         conv2d_39[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_30 (Activation)      (None, None, None, 1 0           batch_normalization_30[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_33 (Activation)      (None, None, None, 1 0           batch_normalization_33[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_38 (Activation)      (None, None, None, 1 0           batch_normalization_38[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_39 (Activation)      (None, None, None, 1 0           batch_normalization_39[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "mixed4 (Concatenate)            (None, None, None, 7 0           activation_30[0][0]              \n",
      "                                                                 activation_33[0][0]              \n",
      "                                                                 activation_38[0][0]              \n",
      "                                                                 activation_39[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_44 (Conv2D)              (None, None, None, 1 122880      mixed4[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_44 (BatchNo (None, None, None, 1 480         conv2d_44[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_44 (Activation)      (None, None, None, 1 0           batch_normalization_44[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_45 (Conv2D)              (None, None, None, 1 179200      activation_44[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_45 (BatchNo (None, None, None, 1 480         conv2d_45[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_45 (Activation)      (None, None, None, 1 0           batch_normalization_45[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_41 (Conv2D)              (None, None, None, 1 122880      mixed4[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_46 (Conv2D)              (None, None, None, 1 179200      activation_45[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_41 (BatchNo (None, None, None, 1 480         conv2d_41[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_46 (BatchNo (None, None, None, 1 480         conv2d_46[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_41 (Activation)      (None, None, None, 1 0           batch_normalization_41[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_46 (Activation)      (None, None, None, 1 0           batch_normalization_46[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_42 (Conv2D)              (None, None, None, 1 179200      activation_41[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_47 (Conv2D)              (None, None, None, 1 179200      activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_42 (BatchNo (None, None, None, 1 480         conv2d_42[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_47 (BatchNo (None, None, None, 1 480         conv2d_47[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_42 (Activation)      (None, None, None, 1 0           batch_normalization_42[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_47 (Activation)      (None, None, None, 1 0           batch_normalization_47[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_4 (AveragePoo (None, None, None, 7 0           mixed4[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_40 (Conv2D)              (None, None, None, 1 147456      mixed4[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_43 (Conv2D)              (None, None, None, 1 215040      activation_42[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_48 (Conv2D)              (None, None, None, 1 215040      activation_47[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_49 (Conv2D)              (None, None, None, 1 147456      average_pooling2d_4[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_40 (BatchNo (None, None, None, 1 576         conv2d_40[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_43 (BatchNo (None, None, None, 1 576         conv2d_43[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_48 (BatchNo (None, None, None, 1 576         conv2d_48[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_49 (BatchNo (None, None, None, 1 576         conv2d_49[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_40 (Activation)      (None, None, None, 1 0           batch_normalization_40[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_43 (Activation)      (None, None, None, 1 0           batch_normalization_43[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_48 (Activation)      (None, None, None, 1 0           batch_normalization_48[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_49 (Activation)      (None, None, None, 1 0           batch_normalization_49[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "mixed5 (Concatenate)            (None, None, None, 7 0           activation_40[0][0]              \n",
      "                                                                 activation_43[0][0]              \n",
      "                                                                 activation_48[0][0]              \n",
      "                                                                 activation_49[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_54 (Conv2D)              (None, None, None, 1 122880      mixed5[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_54 (BatchNo (None, None, None, 1 480         conv2d_54[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_54 (Activation)      (None, None, None, 1 0           batch_normalization_54[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_55 (Conv2D)              (None, None, None, 1 179200      activation_54[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_55 (BatchNo (None, None, None, 1 480         conv2d_55[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_55 (Activation)      (None, None, None, 1 0           batch_normalization_55[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_51 (Conv2D)              (None, None, None, 1 122880      mixed5[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_56 (Conv2D)              (None, None, None, 1 179200      activation_55[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_51 (BatchNo (None, None, None, 1 480         conv2d_51[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_56 (BatchNo (None, None, None, 1 480         conv2d_56[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_51 (Activation)      (None, None, None, 1 0           batch_normalization_51[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_56 (Activation)      (None, None, None, 1 0           batch_normalization_56[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_52 (Conv2D)              (None, None, None, 1 179200      activation_51[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_57 (Conv2D)              (None, None, None, 1 179200      activation_56[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_52 (BatchNo (None, None, None, 1 480         conv2d_52[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_57 (BatchNo (None, None, None, 1 480         conv2d_57[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_52 (Activation)      (None, None, None, 1 0           batch_normalization_52[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_57 (Activation)      (None, None, None, 1 0           batch_normalization_57[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_5 (AveragePoo (None, None, None, 7 0           mixed5[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_50 (Conv2D)              (None, None, None, 1 147456      mixed5[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_53 (Conv2D)              (None, None, None, 1 215040      activation_52[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_58 (Conv2D)              (None, None, None, 1 215040      activation_57[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_59 (Conv2D)              (None, None, None, 1 147456      average_pooling2d_5[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_50 (BatchNo (None, None, None, 1 576         conv2d_50[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_53 (BatchNo (None, None, None, 1 576         conv2d_53[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_58 (BatchNo (None, None, None, 1 576         conv2d_58[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_59 (BatchNo (None, None, None, 1 576         conv2d_59[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_50 (Activation)      (None, None, None, 1 0           batch_normalization_50[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_53 (Activation)      (None, None, None, 1 0           batch_normalization_53[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_58 (Activation)      (None, None, None, 1 0           batch_normalization_58[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_59 (Activation)      (None, None, None, 1 0           batch_normalization_59[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "mixed6 (Concatenate)            (None, None, None, 7 0           activation_50[0][0]              \n",
      "                                                                 activation_53[0][0]              \n",
      "                                                                 activation_58[0][0]              \n",
      "                                                                 activation_59[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_64 (Conv2D)              (None, None, None, 1 147456      mixed6[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_64 (BatchNo (None, None, None, 1 576         conv2d_64[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_64 (Activation)      (None, None, None, 1 0           batch_normalization_64[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_65 (Conv2D)              (None, None, None, 1 258048      activation_64[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_65 (BatchNo (None, None, None, 1 576         conv2d_65[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_65 (Activation)      (None, None, None, 1 0           batch_normalization_65[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_61 (Conv2D)              (None, None, None, 1 147456      mixed6[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_66 (Conv2D)              (None, None, None, 1 258048      activation_65[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_61 (BatchNo (None, None, None, 1 576         conv2d_61[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_66 (BatchNo (None, None, None, 1 576         conv2d_66[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_61 (Activation)      (None, None, None, 1 0           batch_normalization_61[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_66 (Activation)      (None, None, None, 1 0           batch_normalization_66[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_62 (Conv2D)              (None, None, None, 1 258048      activation_61[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_67 (Conv2D)              (None, None, None, 1 258048      activation_66[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_62 (BatchNo (None, None, None, 1 576         conv2d_62[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_67 (BatchNo (None, None, None, 1 576         conv2d_67[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_62 (Activation)      (None, None, None, 1 0           batch_normalization_62[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_67 (Activation)      (None, None, None, 1 0           batch_normalization_67[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_6 (AveragePoo (None, None, None, 7 0           mixed6[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_60 (Conv2D)              (None, None, None, 1 147456      mixed6[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_63 (Conv2D)              (None, None, None, 1 258048      activation_62[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_68 (Conv2D)              (None, None, None, 1 258048      activation_67[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_69 (Conv2D)              (None, None, None, 1 147456      average_pooling2d_6[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_60 (BatchNo (None, None, None, 1 576         conv2d_60[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_63 (BatchNo (None, None, None, 1 576         conv2d_63[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_68 (BatchNo (None, None, None, 1 576         conv2d_68[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_69 (BatchNo (None, None, None, 1 576         conv2d_69[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_60 (Activation)      (None, None, None, 1 0           batch_normalization_60[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_63 (Activation)      (None, None, None, 1 0           batch_normalization_63[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_68 (Activation)      (None, None, None, 1 0           batch_normalization_68[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_69 (Activation)      (None, None, None, 1 0           batch_normalization_69[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "mixed7 (Concatenate)            (None, None, None, 7 0           activation_60[0][0]              \n",
      "                                                                 activation_63[0][0]              \n",
      "                                                                 activation_68[0][0]              \n",
      "                                                                 activation_69[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_72 (Conv2D)              (None, None, None, 1 147456      mixed7[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_72 (BatchNo (None, None, None, 1 576         conv2d_72[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_72 (Activation)      (None, None, None, 1 0           batch_normalization_72[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_73 (Conv2D)              (None, None, None, 1 258048      activation_72[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_73 (BatchNo (None, None, None, 1 576         conv2d_73[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_73 (Activation)      (None, None, None, 1 0           batch_normalization_73[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_70 (Conv2D)              (None, None, None, 1 147456      mixed7[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_74 (Conv2D)              (None, None, None, 1 258048      activation_73[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_70 (BatchNo (None, None, None, 1 576         conv2d_70[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_74 (BatchNo (None, None, None, 1 576         conv2d_74[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_70 (Activation)      (None, None, None, 1 0           batch_normalization_70[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_74 (Activation)      (None, None, None, 1 0           batch_normalization_74[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_71 (Conv2D)              (None, None, None, 3 552960      activation_70[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_75 (Conv2D)              (None, None, None, 1 331776      activation_74[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_71 (BatchNo (None, None, None, 3 960         conv2d_71[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_75 (BatchNo (None, None, None, 1 576         conv2d_75[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_71 (Activation)      (None, None, None, 3 0           batch_normalization_71[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_75 (Activation)      (None, None, None, 1 0           batch_normalization_75[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)  (None, None, None, 7 0           mixed7[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "mixed8 (Concatenate)            (None, None, None, 1 0           activation_71[0][0]              \n",
      "                                                                 activation_75[0][0]              \n",
      "                                                                 max_pooling2d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_80 (Conv2D)              (None, None, None, 4 573440      mixed8[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_80 (BatchNo (None, None, None, 4 1344        conv2d_80[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_80 (Activation)      (None, None, None, 4 0           batch_normalization_80[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_77 (Conv2D)              (None, None, None, 3 491520      mixed8[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_81 (Conv2D)              (None, None, None, 3 1548288     activation_80[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_77 (BatchNo (None, None, None, 3 1152        conv2d_77[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_81 (BatchNo (None, None, None, 3 1152        conv2d_81[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_77 (Activation)      (None, None, None, 3 0           batch_normalization_77[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_81 (Activation)      (None, None, None, 3 0           batch_normalization_81[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_78 (Conv2D)              (None, None, None, 3 442368      activation_77[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_79 (Conv2D)              (None, None, None, 3 442368      activation_77[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_82 (Conv2D)              (None, None, None, 3 442368      activation_81[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_83 (Conv2D)              (None, None, None, 3 442368      activation_81[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_7 (AveragePoo (None, None, None, 1 0           mixed8[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_76 (Conv2D)              (None, None, None, 3 409600      mixed8[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_78 (BatchNo (None, None, None, 3 1152        conv2d_78[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_79 (BatchNo (None, None, None, 3 1152        conv2d_79[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_82 (BatchNo (None, None, None, 3 1152        conv2d_82[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_83 (BatchNo (None, None, None, 3 1152        conv2d_83[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_84 (Conv2D)              (None, None, None, 1 245760      average_pooling2d_7[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_76 (BatchNo (None, None, None, 3 960         conv2d_76[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_78 (Activation)      (None, None, None, 3 0           batch_normalization_78[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_79 (Activation)      (None, None, None, 3 0           batch_normalization_79[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_82 (Activation)      (None, None, None, 3 0           batch_normalization_82[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_83 (Activation)      (None, None, None, 3 0           batch_normalization_83[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_84 (BatchNo (None, None, None, 1 576         conv2d_84[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_76 (Activation)      (None, None, None, 3 0           batch_normalization_76[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "mixed9_0 (Concatenate)          (None, None, None, 7 0           activation_78[0][0]              \n",
      "                                                                 activation_79[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, None, None, 7 0           activation_82[0][0]              \n",
      "                                                                 activation_83[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_84 (Activation)      (None, None, None, 1 0           batch_normalization_84[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "mixed9 (Concatenate)            (None, None, None, 2 0           activation_76[0][0]              \n",
      "                                                                 mixed9_0[0][0]                   \n",
      "                                                                 concatenate[0][0]                \n",
      "                                                                 activation_84[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_89 (Conv2D)              (None, None, None, 4 917504      mixed9[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_89 (BatchNo (None, None, None, 4 1344        conv2d_89[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_89 (Activation)      (None, None, None, 4 0           batch_normalization_89[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_86 (Conv2D)              (None, None, None, 3 786432      mixed9[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_90 (Conv2D)              (None, None, None, 3 1548288     activation_89[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_86 (BatchNo (None, None, None, 3 1152        conv2d_86[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_90 (BatchNo (None, None, None, 3 1152        conv2d_90[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_86 (Activation)      (None, None, None, 3 0           batch_normalization_86[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_90 (Activation)      (None, None, None, 3 0           batch_normalization_90[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_87 (Conv2D)              (None, None, None, 3 442368      activation_86[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_88 (Conv2D)              (None, None, None, 3 442368      activation_86[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_91 (Conv2D)              (None, None, None, 3 442368      activation_90[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_92 (Conv2D)              (None, None, None, 3 442368      activation_90[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_8 (AveragePoo (None, None, None, 2 0           mixed9[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_85 (Conv2D)              (None, None, None, 3 655360      mixed9[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_87 (BatchNo (None, None, None, 3 1152        conv2d_87[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_88 (BatchNo (None, None, None, 3 1152        conv2d_88[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_91 (BatchNo (None, None, None, 3 1152        conv2d_91[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_92 (BatchNo (None, None, None, 3 1152        conv2d_92[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_93 (Conv2D)              (None, None, None, 1 393216      average_pooling2d_8[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_85 (BatchNo (None, None, None, 3 960         conv2d_85[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_87 (Activation)      (None, None, None, 3 0           batch_normalization_87[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_88 (Activation)      (None, None, None, 3 0           batch_normalization_88[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_91 (Activation)      (None, None, None, 3 0           batch_normalization_91[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_92 (Activation)      (None, None, None, 3 0           batch_normalization_92[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_93 (BatchNo (None, None, None, 1 576         conv2d_93[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_85 (Activation)      (None, None, None, 3 0           batch_normalization_85[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "mixed9_1 (Concatenate)          (None, None, None, 7 0           activation_87[0][0]              \n",
      "                                                                 activation_88[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, None, None, 7 0           activation_91[0][0]              \n",
      "                                                                 activation_92[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_93 (Activation)      (None, None, None, 1 0           batch_normalization_93[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "mixed10 (Concatenate)           (None, None, None, 2 0           activation_85[0][0]              \n",
      "                                                                 mixed9_1[0][0]                   \n",
      "                                                                 concatenate_1[0][0]              \n",
      "                                                                 activation_93[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d (Globa (None, 2048)         0           mixed10[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1024)         2098176     global_average_pooling2d[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 1024)         0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 128)          131200      dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 128)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 2)            258         dropout_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 24,032,418\n",
      "Trainable params: 2,229,634\n",
      "Non-trainable params: 21,802,784\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 22;\n",
       "                var nbb_unformatted_code = \"# let's visualize layer names and layer indices to see how many layers\\n# we should freeze:\\nmodel.summary()\\n# for i, layer in enumerate(model.layers):\\n#    print(i, layer.name)\";\n",
       "                var nbb_formatted_code = \"# let's visualize layer names and layer indices to see how many layers\\n# we should freeze:\\nmodel.summary()\\n# for i, layer in enumerate(model.layers):\\n#    print(i, layer.name)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# let's visualize layer names and layer indices to see how many layers\n",
    "# we should freeze:\n",
    "model.summary()\n",
    "# for i, layer in enumerate(model.layers):\n",
    "#    print(i, layer.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 23;\n",
       "                var nbb_unformatted_code = \"# Let's unfreeze the whole model\\nfor layer in model.layers:\\n    layer.trainable = True\\n# Let's build an optimizer\\noptimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)\\nes = EarlyStopping(\\n    monitor=\\\"val_loss\\\",\\n    verbose=1,\\n    mode=\\\"min\\\",\\n    patience=patience,\\n    restore_best_weights=True,\\n)\\n# We need to recompile the model for these modifications to take effect\\nes.set_model(model)\\nmodel.compile(optimizer=optimizer, loss=\\\"binary_crossentropy\\\")  # , metrics=[\\\"AUC\\\"])\";\n",
       "                var nbb_formatted_code = \"# Let's unfreeze the whole model\\nfor layer in model.layers:\\n    layer.trainable = True\\n# Let's build an optimizer\\noptimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)\\nes = EarlyStopping(\\n    monitor=\\\"val_loss\\\",\\n    verbose=1,\\n    mode=\\\"min\\\",\\n    patience=patience,\\n    restore_best_weights=True,\\n)\\n# We need to recompile the model for these modifications to take effect\\nes.set_model(model)\\nmodel.compile(optimizer=optimizer, loss=\\\"binary_crossentropy\\\")  # , metrics=[\\\"AUC\\\"])\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Let's unfreeze the whole model\n",
    "for layer in model.layers:\n",
    "    layer.trainable = True\n",
    "# Let's build an optimizer\n",
    "optimizer = Lookahead(RectifiedAdam(), sync_period=6, slow_step_size=0.5)\n",
    "es = EarlyStopping(\n",
    "    monitor=\"val_loss\",\n",
    "    verbose=1,\n",
    "    mode=\"min\",\n",
    "    patience=patience,\n",
    "    restore_best_weights=True,\n",
    ")\n",
    "# We need to recompile the model for these modifications to take effect\n",
    "es.set_model(model)\n",
    "model.compile(optimizer=optimizer, loss=\"binary_crossentropy\")  # , metrics=[\"AUC\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train for 230.0 steps, validate for 29.0 steps\n",
      "Epoch 1/200\n",
      "230/230 [==============================] - 148s 645ms/step - loss: 0.1750 - val_loss: 0.1671\n",
      "Epoch 2/200\n",
      "230/230 [==============================] - 75s 324ms/step - loss: 0.1498 - val_loss: 0.1390\n",
      "Epoch 3/200\n",
      "230/230 [==============================] - 120s 520ms/step - loss: 0.1350 - val_loss: 0.1889\n",
      "Epoch 4/200\n",
      "212/230 [==========================>...] - ETA: 6s - loss: 0.1291"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# we train our model again (this time fine-tuning the top 2 inception blocks\n",
    "# alongside the top Dense layers\n",
    "history_2 = model.fit(\n",
    "    dataset, \n",
    "    callbacks=[es],\n",
    "    epochs=epochs_2,\n",
    "    steps_per_epoch=steps_per_epoch,\n",
    "    validation_data=dataset_valid,\n",
    "    validation_steps=steps_per_epoch_val\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_metric(history_2, \"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot_metric(history_2, \"AUC\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "valid_data = preprocess_data(X[valid_indices], fixed_text_to_square_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "preds = model.predict(valid_data)\n",
    "print(roc_auc_score(Y[valid_indices], preds[:, 1]))\n",
    "del valid_data\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "test_data = preprocess_data(X[test_indices], fixed_text_to_square_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = model.predict(test_data)\n",
    "print(roc_auc_score(Y[test_indices], preds[:, 1]))\n",
    "del test_data\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://medium.com/google-developer-experts/interpreting-deep-learning-models-for-computer-vision-f95683e23c1d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}