get_model_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "from matplotlib.ticker import (MultipleLocator, FormatStrFormatter)\n",
    "from dataclasses import dataclass \n",
    "\n",
    "block_plot = False\n",
    "plt.rcParams['figure.figsize'] = (12, 9)\n",
    "SEED_VALUE = 42 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "@dataclass(frozen=True)\n",
    "class DatasetConfig:\n",
    "    NUM_CLASSES: int = 29\n",
    "    IMG_HEIGHT:  int = 224\n",
    "    IMG_WIDTH:   int = 224\n",
    "    CHANNELS:    int = 3\n",
    "    BATCH_SIZE:  int = 32\n",
    "    DATA_ROOT:   str = './dataset_ASL_150'\n",
    "        \n",
    "@dataclass(frozen=True)\n",
    "class TrainingConfig:\n",
    "    BATCH_SIZE:     int   = 32\n",
    "    EPOCHS:         int   = 51\n",
    "    LEARNING_RATE:  float = 0.0001\n",
    "    CHECKPOINT_DIR: str   = './saved_models_asl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the Model.\n",
    "training_results = vgg16_model.fit(train_dataset,\n",
    "                                   validation_data=valid_dataset,\n",
    "                                   epochs=TrainingConfig.EPOCHS,\n",
    "                                   workers=4,\n",
    "                                   use_multiprocessing=True, \n",
    "                                   callbacks=model_checkpoint_callback,\n",
    "                                  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot the training and validation accuracy and loss over the epochs.\n",
    "\n",
    "def plot_results(metrics, ylabel=None, ylim=None, metric_name=None, color=None):\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(15, 4))\n",
    "\n",
    "    if not (isinstance(metric_name, list) or isinstance(metric_name, tuple)):\n",
    "        metrics = [metrics,]\n",
    "        metric_name = [metric_name,]\n",
    "        \n",
    "    for idx, metric in enumerate(metrics):    \n",
    "        ax.plot(metric, color=color[idx])\n",
    "    \n",
    "    plt.xlabel(\"Epoch\")\n",
    "    plt.ylabel(ylabel)\n",
    "    plt.title(ylabel)\n",
    "    plt.xlim([0, TrainingConfig.EPOCHS-1])\n",
    "    plt.ylim(ylim)\n",
    "    # Tailor x-axis tick marks\n",
    "    ax.xaxis.set_major_locator(MultipleLocator(5))\n",
    "    ax.xaxis.set_major_formatter(FormatStrFormatter('%d'))\n",
    "    ax.xaxis.set_minor_locator(MultipleLocator(1))\n",
    "    plt.grid(True)\n",
    "    plt.legend(metric_name)   \n",
    "    plt.show(block=block_plot)\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "# Retrieve training results.\n",
    "train_loss = training_results.history[\"loss\"]\n",
    "train_acc  = training_results.history[\"accuracy\"]\n",
    "valid_loss = training_results.history[\"val_loss\"]\n",
    "valid_acc  = training_results.history[\"val_accuracy\"]\n",
    "    \n",
    "plot_results([ train_acc, valid_acc ], \n",
    "            ylabel=\"Accuracy\",\n",
    "            ylim = [0.5, 1.0],\n",
    "            metric_name=[\"Training Accuracy\", \"Validation Accuracy\"],\n",
    "            color=[\"g\", \"b\"])\n",
    " \n",
    "max_loss = 2.0\n",
    "\n",
    "plot_results([ train_loss, valid_loss ],        \n",
    "            ylabel=\"Loss\", \n",
    "            ylim = [0.0, max_loss],\n",
    "            metric_name=[\"Training Loss\", \"Validation Loss\"],\n",
    "            color=[\"g\", \"b\"]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load saved model.\n",
    "# Final evaluation accuracy on the validation set.\n",
    "model = tf.keras.models.load_model(TrainingConfig.CHECKPOINT_DIR)\n",
    "print(f\"Model evaluation accuracy: {model.evaluate(valid_dataset)[1]*100.:.3f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf_gpu",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.9.7 (default, Sep 16 2021, 16:59:28) [MSC v.1916 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "07ca2b236c0269e8e97825cf335cfca31b1a5fed1dbdb20dbc94341f784cd392"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}