AlexNet_AML.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "9838132cc3be4902b417294203b86b64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_65f36d4c74874ac28c09aa72c496ae96",
              "IPY_MODEL_48602777a3344929aced61e7f9e0f5be",
              "IPY_MODEL_9423aaffbc914747b3958aef4e45cf17"
            ],
            "layout": "IPY_MODEL_b9a0ed55a66044b19800808683fcb0a6"
          }
        },
        "65f36d4c74874ac28c09aa72c496ae96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_913ef234ce694ae5a7b75b97e2bf6a3c",
            "placeholder": "​",
            "style": "IPY_MODEL_0214102239f748dbb8278b5e9d9da9e1",
            "value": "100%"
          }
        },
        "48602777a3344929aced61e7f9e0f5be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e6d4e27588f6413189be30a20b972849",
            "max": 244408911,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ce46467099c8493ea9daf3be9ee45905",
            "value": 244408911
          }
        },
        "9423aaffbc914747b3958aef4e45cf17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_15a8b0c45e4a4c3bb020984a6a46683a",
            "placeholder": "​",
            "style": "IPY_MODEL_0da11d0ecd4446d4a2edcfe224037037",
            "value": " 233M/233M [00:01&lt;00:00, 218MB/s]"
          }
        },
        "b9a0ed55a66044b19800808683fcb0a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "913ef234ce694ae5a7b75b97e2bf6a3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0214102239f748dbb8278b5e9d9da9e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e6d4e27588f6413189be30a20b972849": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce46467099c8493ea9daf3be9ee45905": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "15a8b0c45e4a4c3bb020984a6a46683a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0da11d0ecd4446d4a2edcfe224037037": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fo942LMOdlh4"
      },
      "source": [
        "**Import libraries**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DokFOdD1dJEl"
      },
      "source": [
        "import os\n",
        "import os.path\n",
        "import sys\n",
        "\n",
        "import logging\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Subset, DataLoader\n",
        "from torch.backends import cudnn\n",
        "\n",
        "import torchvision\n",
        "from torchvision import transforms\n",
        "from torchvision.models import alexnet\n",
        "from torchvision.models import AlexNet_Weights\n",
        "from torchvision.datasets import VisionDataset\n",
        "from PIL import Image\n",
        "from tqdm import tqdm\n",
        "\n",
        "import numpy as np"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OIDLJuIXK_vh"
      },
      "source": [
        "**Set Arguments**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d5PkYfqfK_SA"
      },
      "source": [
        "DEVICE = 'cuda' # 'cuda' or 'cpu'\n",
        "\n",
        "NUM_CLASSES = 102 # 101 + 1: There is am extra Background class that should be removed \n",
        "\n",
        "BATCH_SIZE = 256     # Higher batch sizes allows for larger learning rates. An empirical heuristic suggests that, when changing\n",
        "                     # the batch size, learning rate should change by the same factor to have comparable results\n",
        "\n",
        "LR = 0.01         # The initial Learning Rate\n",
        "MOMENTUM = 0.9       # Hyperparameter for SGD, keep this at 0.9 when using SGD\n",
        "WEIGHT_DECAY = 5e-5  # Regularization, you can keep this at the default\n",
        "\n",
        "NUM_EPOCHS = 30      # Total number of training epochs (iterations over dataset)\n",
        "STEP_SIZE = 20       # How many epochs before decreasing learning rate (if using a step-down policy)\n",
        "GAMMA = 0.1          # Multiplicative factor for learning rate step-down\n",
        "\n",
        "\n",
        "LOG_FREQUENCY = 10"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Define Caltech**"
      ],
      "metadata": {
        "id": "U0ertfOdSBmm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def pil_loader(path):\n",
        "    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)\n",
        "    with open(path, 'rb') as f:\n",
        "        img = Image.open(f)\n",
        "        return img.convert('RGB')\n",
        "\n",
        "  \n",
        "class Caltech(VisionDataset):\n",
        "    def __init__(self, root, split='train', transform=None, target_transform=None):\n",
        "        super(Caltech, self).__init__(root, transform=transform, target_transform=target_transform)\n",
        "\n",
        "        self.split = split # This defines the split you are going to use\n",
        "                           # (split files are called 'train.txt' and 'test.txt')\n",
        "\n",
        "        self.lst = list()\n",
        "        self.labels = dict()\n",
        "        self.nlabels = 0\n",
        "\n",
        "        with open(\"Caltech101/\"+self.split+\".txt\") as f:\n",
        "          for path in f.readlines():\n",
        "            label, img = path.strip().split('/')\n",
        "            if label!=\"BACKGROUND_Google\":\n",
        "              if label not in self.labels: \n",
        "                self.labels[label] = self.nlabels\n",
        "                self.nlabels += 1\n",
        "            \n",
        "              self.lst.append((pil_loader(os.path.join(root, path.strip())), self.labels[label]))\n",
        "\n",
        "\n",
        "        '''\n",
        "        - Here you should implement the logic for reading the splits files and accessing elements\n",
        "        - If the RAM size allows it, it is faster to store all data in memory\n",
        "        - PyTorch Dataset classes use indexes to read elements\n",
        "        - You should provide a way for the __getitem__ method to access the image-label pair\n",
        "          through the index\n",
        "        - Labels should start from 0, so for Caltech you will have lables 0...100 (excluding the background class) \n",
        "        '''\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        '''\n",
        "        __getitem__ should access an element through its index\n",
        "        Args:\n",
        "            index (int): Index\n",
        "\n",
        "        Returns:\n",
        "            tuple: (sample, target) where target is class_index of the target class.\n",
        "        '''\n",
        "\n",
        "        image, label = self.lst[index]\n",
        "\n",
        "        # Applies preprocessing when accessing the image\n",
        "        if self.transform is not None:\n",
        "            image = self.transform(image)\n",
        "\n",
        "        return image, label\n",
        "\n",
        "    def __len__(self):\n",
        "        '''\n",
        "        The __len__ method returns the length of the dataset\n",
        "        It is mandatory, as this is used by several other components\n",
        "        '''\n",
        "        length = len(self.lst) # Provide a way to get the length (number of elements) of the dataset\n",
        "        return length"
      ],
      "metadata": {
        "id": "t4etzFcBR-x3"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9gwii0TBHvzh"
      },
      "source": [
        "**Define Data Preprocessing**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QUDdw4j2H0Mc"
      },
      "source": [
        "# Define transforms for training phase\n",
        "train_transform = transforms.Compose([transforms.Resize(256),      # Resizes short size of the PIL image to 256\n",
        "                                      transforms.CenterCrop(224),  # Crops a central square patch of the image\n",
        "                                                                   # 224 because torchvision's AlexNet needs a 224x224 input!\n",
        "                                                                   # Remember this when applying different transformations, otherwise you get an error\n",
        "                                      transforms.RandomHorizontalFlip([0.5]),\n",
        "                                      transforms.\n",
        "                                      transforms.ToTensor(), # Turn PIL Image to torch.Tensor\n",
        "                                      transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) # Normalizes tensor with mean and standard deviation\n",
        "])\n",
        "# Define transforms for the evaluation phase\n",
        "eval_transform = transforms.Compose([transforms.Resize(256),\n",
        "                                      transforms.CenterCrop(224),\n",
        "                                      transforms.ToTensor(),\n",
        "                                      transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))                                    \n",
        "])"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2qYIHPzYLY7i"
      },
      "source": [
        "**Prepare Dataset**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QfVq_uDHLbsR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a0e1a68b-192b-4fe0-a2dc-23f74861906e"
      },
      "source": [
        "# Clone github repository with data\n",
        "if not os.path.isdir('./Caltech101'):\n",
        "  !git clone https://github.com/MachineLearning2020/Homework2-Caltech101.git\n",
        "  !mv 'Homework2-Caltech101' 'Caltech101'\n",
        "\n",
        "DATA_DIR = 'Caltech101/101_ObjectCategories'\n",
        "# Prepare Pytorch train/test Datasets\n",
        "train_dataset = Caltech(DATA_DIR, split='train',  transform=train_transform)\n",
        "test_dataset = Caltech(DATA_DIR, split='test', transform=eval_transform)\n",
        "\n",
        "length = len(train_dataset)\n",
        "indices = np.arange(length)\n",
        "np.random.shuffle(indices)\n",
        "\n",
        "train_indexes = indices[:int(length*2/3)]\n",
        "val_indexes = indices[int(length*2/3):]\n",
        "\n",
        "val_dataset = Subset(train_dataset, val_indexes)\n",
        "train_dataset = Subset(train_dataset, train_indexes)\n",
        "\n",
        "# Check dataset sizes\n",
        "print('Train Dataset: {}'.format(len(train_dataset)))\n",
        "print('Valid Dataset: {}'.format(len(val_dataset)))\n",
        "print('Test Dataset: {}'.format(len(test_dataset)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train Dataset: 3856\n",
            "Valid Dataset: 1928\n",
            "Test Dataset: 2893\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FYEDQ7Z21ldN"
      },
      "source": [
        "**Prepare Dataloaders**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VriRw8SI1nle",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f7880b12-c9e9-4c5e-a418-582cda6e7c22"
      },
      "source": [
        "# Dataloaders iterate over pytorch datasets and transparently provide useful functions (e.g. parallelization and shuffling)\n",
        "train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, drop_last=True)\n",
        "val_dataloader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\n",
        "\n",
        "test_dataloader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:566: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gbZ1t5Qs2z4j"
      },
      "source": [
        "**Prepare Network**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "exHUjtXa22DN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124,
          "referenced_widgets": [
            "9838132cc3be4902b417294203b86b64",
            "65f36d4c74874ac28c09aa72c496ae96",
            "48602777a3344929aced61e7f9e0f5be",
            "9423aaffbc914747b3958aef4e45cf17",
            "b9a0ed55a66044b19800808683fcb0a6",
            "913ef234ce694ae5a7b75b97e2bf6a3c",
            "0214102239f748dbb8278b5e9d9da9e1",
            "e6d4e27588f6413189be30a20b972849",
            "ce46467099c8493ea9daf3be9ee45905",
            "15a8b0c45e4a4c3bb020984a6a46683a",
            "0da11d0ecd4446d4a2edcfe224037037"
          ]
        },
        "outputId": "64dda58d-96a1-4493-98bc-831b223df1da"
      },
      "source": [
        "net = alexnet(weights=(\"pretrained\", AlexNet_Weights.IMAGENET1K_V1))# Loading AlexNet model\n",
        "#net = alexnet() #WITHOUT IMAGENET WEIGHTS\n",
        "\n",
        "# AlexNet has 1000 output neurons, corresponding to the 1000 ImageNet's classes\n",
        "# We need 101 outputs for Caltech-101\n",
        "net.classifier[6] = nn.Linear(4096, NUM_CLASSES) # nn.Linear in pytorch is a fully connected layer\n",
        "                                                 # The convolutional layer is nn.Conv2d\n",
        "\n",
        "# We just changed the last layer of AlexNet with a new fully connected layer with 101 outputs\n",
        "# It is strongly suggested to study torchvision.models.alexnet source code"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and will be removed in 0.15. The current behavior is equivalent to passing `weights=AlexNet_Weights.IMAGENET1K_V1`. You can also use `weights=AlexNet_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/alexnet-owt-7be5be79.pth\" to /root/.cache/torch/hub/checkpoints/alexnet-owt-7be5be79.pth\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0.00/233M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9838132cc3be4902b417294203b86b64"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KEyL3H_R4qCf"
      },
      "source": [
        "**Prepare Training**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9sjq00G94tSc"
      },
      "source": [
        "# Define loss function\n",
        "criterion = nn.CrossEntropyLoss() # for classification, we use Cross Entropy\n",
        "\n",
        "# Choose parameters to optimize\n",
        "# To access a different set of parameters, you have to access submodules of AlexNet\n",
        "# (nn.Module objects, like AlexNet, implement the Composite Pattern)\n",
        "# e.g.: parameters of the fully connected layers: net.classifier.parameters()\n",
        "# e.g.: parameters of the convolutional layers: look at alexnet's source code ;) \n",
        "parameters_to_optimize = net.parameters() # In this case we optimize over all the parameters of AlexNet\n",
        "#parameters_to_optimize = net.classifier.parameters() #We optimize only fully connected layers\n",
        "#parameters_to_optimize = net.features.parameters() #We optimize only convolutional layers\n",
        "\n",
        "# Define optimizer\n",
        "# An optimizer updates the weights based on loss\n",
        "# We use SGD with momentum\n",
        "optimizer = optim.SGD(parameters_to_optimize, lr=LR, momentum=MOMENTUM, weight_decay=WEIGHT_DECAY)\n",
        "\n",
        "# Define scheduler\n",
        "# A scheduler dynamically changes learning rate\n",
        "# The most common schedule is the step(-down), which multiplies learning rate by gamma every STEP_SIZE epochs\n",
        "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=STEP_SIZE, gamma=GAMMA)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AxYUli9d9uYQ"
      },
      "source": [
        "**Train**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZcoQ5fD49yT_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 490
        },
        "outputId": "15584e57-2597-4664-b33a-f249eb366f51"
      },
      "source": [
        "# By default, everything is loaded to cpu\n",
        "net = net.to(DEVICE) # this will bring the network to GPU if DEVICE is cuda\n",
        "\n",
        "cudnn.benchmark # Calling this optimizes runtime\n",
        "\n",
        "current_step = 0\n",
        "# Start iterating over the epochs\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "  print('Starting epoch {}/{}, LR = {}'.format(epoch+1, NUM_EPOCHS, scheduler.get_lr()))\n",
        "\n",
        "  # Iterate over the dataset\n",
        "  for images, labels in train_dataloader:\n",
        "    # Bring data over the device of choice\n",
        "    images = images.to(DEVICE)\n",
        "    labels = labels.to(DEVICE)\n",
        "\n",
        "    net.train() # Sets module in training mode\n",
        "\n",
        "    # PyTorch, by default, accumulates gradients after each backward pass\n",
        "    # We need to manually set the gradients to zero before starting a new iteration\n",
        "    optimizer.zero_grad() # Zero-ing the gradients\n",
        "\n",
        "    # Forward pass to the network\n",
        "    outputs = net(images)\n",
        "\n",
        "    # Compute loss based on output and ground truth\n",
        "    loss = criterion(outputs, labels)\n",
        "\n",
        "    # Log loss\n",
        "    if current_step % LOG_FREQUENCY == 0:\n",
        "      print('Step {}, Loss {}'.format(current_step, loss.item()))\n",
        "\n",
        "    # Compute gradients for each layer and update weights\n",
        "    loss.backward()  # backward pass: computes gradients\n",
        "    optimizer.step() # update weights based on accumulated gradients\n",
        "\n",
        "    current_step += 1\n",
        "\n",
        "  # Step the scheduler\n",
        "  scheduler.step() "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Starting epoch 1/30, LR = [0.01]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/optim/lr_scheduler.py:372: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
            "  \"please use `get_last_lr()`.\", UserWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Step 0, Loss 4.94946813583374\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-18-f7bec9abfe6f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m   \u001b[0;31m# Iterate over the dataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m   \u001b[0;32mfor\u001b[0m \u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m     \u001b[0;31m# Bring data over the device of choice\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0mimages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDEVICE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    679\u001b[0m                 \u001b[0;31m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    680\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 681\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    682\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    683\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1357\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1358\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shutdown\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tasks_outstanding\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1359\u001b[0;31m             \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1360\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tasks_outstanding\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1361\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_get_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1323\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1324\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1325\u001b[0;31m                 \u001b[0msuccess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_get_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1326\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0msuccess\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1327\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_try_get_data\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1161\u001b[0m         \u001b[0;31m#   (bool: whether successfully get data, any: data if successful else None)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1162\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1163\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data_queue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1164\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1165\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/multiprocessing/queues.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m    102\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mblock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m                     \u001b[0mtimeout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdeadline\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmonotonic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 104\u001b[0;31m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_poll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m                         \u001b[0;32mraise\u001b[0m \u001b[0mEmpty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m                 \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_poll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/multiprocessing/connection.py\u001b[0m in \u001b[0;36mpoll\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    255\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_closed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    256\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_readable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 257\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_poll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    259\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__enter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/multiprocessing/connection.py\u001b[0m in \u001b[0;36m_poll\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    412\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    413\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_poll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 414\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    415\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    416\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/multiprocessing/connection.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(object_list, timeout)\u001b[0m\n\u001b[1;32m    919\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    920\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 921\u001b[0;31m                 \u001b[0mready\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mselector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    922\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mready\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    923\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfileobj\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevents\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mready\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/selectors.py\u001b[0m in \u001b[0;36mselect\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    413\u001b[0m         \u001b[0mready\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    414\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 415\u001b[0;31m             \u001b[0mfd_event_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_selector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    416\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mInterruptedError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    417\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mready\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UsHFI-GAJd69"
      },
      "source": [
        "**Validation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EO3HV5pqJg1o"
      },
      "source": [
        "net = net.to(DEVICE) # this will bring the network to GPU if DEVICE is cuda\n",
        "net.train(False) # Set Network to evaluation mode\n",
        "\n",
        "running_corrects = 0\n",
        "for images, labels in tqdm(val_dataloader):\n",
        "  images = images.to(DEVICE)\n",
        "  labels = labels.to(DEVICE)\n",
        "\n",
        "  # Forward Pass\n",
        "  outputs = net(images)\n",
        "\n",
        "  # Get predictions\n",
        "  _, preds = torch.max(outputs.data, 1)\n",
        "\n",
        "  # Update Corrects\n",
        "  running_corrects += torch.sum(preds == labels.data).data.item()\n",
        "\n",
        "# Calculate Accuracy\n",
        "accuracy = running_corrects / float(len(val_dataset))\n",
        "\n",
        "print('Validation Accuracy: {}'.format(accuracy))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jxekmR745ySe"
      },
      "source": [
        "**Test**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fSHcUqLB5yWO"
      },
      "source": [
        "net = net.to(DEVICE) # this will bring the network to GPU if DEVICE is cuda\n",
        "net.train(False) # Set Network to evaluation mode\n",
        "\n",
        "running_corrects = 0\n",
        "for images, labels in tqdm(test_dataloader):\n",
        "  images = images.to(DEVICE)\n",
        "  labels = labels.to(DEVICE)\n",
        "\n",
        "  # Forward Pass\n",
        "  outputs = net(images)\n",
        "\n",
        "  # Get predictions\n",
        "  _, preds = torch.max(outputs.data, 1)\n",
        "\n",
        "  # Update Corrects\n",
        "  running_corrects += torch.sum(preds == labels.data).data.item()\n",
        "\n",
        "# Calculate Accuracy\n",
        "accuracy = running_corrects / float(len(test_dataset))\n",
        "\n",
        "print('\\nTest Accuracy: {}'.format(accuracy))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Results\n",
        "**LR = 1e-3, No pre-trained weights**\n",
        "\n",
        "-   \n",
        "\n",
        "-   \n",
        "\n",
        "**LR = 0.01, No pre-trained weights**\n",
        "\n",
        "-   \n",
        "\n",
        "-   \n",
        "\n",
        "**LR = 1e-3, ImageNet weights**\n",
        "\n",
        "-   \n",
        "\n",
        "-   \n",
        "\n",
        "**LR = 0.01, ImageNet weights**\n",
        "\n",
        "- Validation Accuracy: 0.8615145228215768,\n",
        "\n",
        "- Test Accuracy: 0.8565502938126512\n",
        "\n",
        "**LR = 1e-3, ImageNet weights, freeze fully connected layers**\n",
        "\n",
        "-   \n",
        "\n",
        "-   \n",
        "\n",
        "**LR = 1e-3, ImageNet weights, freeze convolutional layers**\n",
        "\n",
        "-   \n",
        "\n",
        "-    \n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "cVJudsdpPlng"
      }
    }
  ]
}