6_SEResNet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-10-13T10:10:12.970182Z",
     "start_time": "2024-10-13T10:10:10.537793Z"
    }
   },
   "source": [
    "import torch\n",
    "from torch import nn, optim\n",
    "from torch.utils.data import random_split, DataLoader\n",
    "from torchinfo import summary\n",
    "from torchvision import datasets, transforms, models\n",
    "\n",
    "DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "print(DEVICE)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## Implementatation",
   "id": "6728b1ca1cd1e4dc"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T10:10:12.990023Z",
     "start_time": "2024-10-13T10:10:12.985495Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class SEBlock(nn.Module):\n",
    "    def __init__(self, channels, reduction_ratio):\n",
    "        super().__init__()\n",
    "        self.gap = nn.AdaptiveAvgPool2d(1)\n",
    "        self.linear1 = nn.Linear(channels, channels//reduction_ratio, bias=False)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.linear2 = nn.Linear(channels//reduction_ratio, channels, bias=False)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.gap(x)\n",
    "        x = torch.squeeze(x)\n",
    "        x = self.linear1(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.linear2(x)\n",
    "        x = self.sigmoid(x)\n",
    "        return x"
   ],
   "id": "b7e4ae5dca804124",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T10:10:13.181655Z",
     "start_time": "2024-10-13T10:10:13.173539Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class ResidualBlock(nn.Module):\n",
    "    def __init__(self, input_channels, inner_channels, reduction_ratio, stride=1, projection=None):\n",
    "        super().__init__()\n",
    "        self.residual_block = nn.Sequential(\n",
    "            nn.Conv2d(input_channels, inner_channels, kernel_size=1, bias=False),\n",
    "            nn.BatchNorm2d(inner_channels),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(inner_channels, inner_channels, kernel_size=3, stride=stride, padding=1, bias=False),\n",
    "            nn.BatchNorm2d(inner_channels),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(inner_channels, inner_channels*4, kernel_size=1, bias=False),\n",
    "            nn.BatchNorm2d(inner_channels*4),\n",
    "        )\n",
    "        self.se_block = SEBlock(inner_channels*4, reduction_ratio)\n",
    "        self.projection = projection\n",
    "        self.relu = nn.ReLU()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        residual = self.residual_block(x)\n",
    "        residual *= self.se_block(residual)[..., None, None]\n",
    "        shortcut = self.projection(x) if self.projection else x\n",
    "        residual = self.relu(residual + shortcut)\n",
    "        return residual\n",
    "\n",
    "\n",
    "class SEResNet(nn.Module):\n",
    "    def __init__(self, n_channels, n_classes, reduction_ratio):\n",
    "        super().__init__()\n",
    "        self.reduction_ratio = reduction_ratio\n",
    "        self.conv1 = nn.Sequential(\n",
    "            nn.Conv2d(n_channels, 64, kernel_size=7, stride=2, padding=3, bias=False),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(),\n",
    "        )\n",
    "        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
    "        self.layers1 = self.make_layers(64, 64, 3, 1)\n",
    "        self.layers2 = self.make_layers(256, 128, 4, 2)\n",
    "        self.layers3 = self.make_layers(512, 256, 6, 2)\n",
    "        self.layers4 = self.make_layers(1024, 512, 3, 2)\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Flatten(),\n",
    "            nn.Linear(2048, n_classes)\n",
    "        )\n",
    "    \n",
    "    def make_layers(self, input_channels, inner_channels, n_blocks, stride):\n",
    "        projection = nn.Sequential(\n",
    "            nn.Conv2d(input_channels, inner_channels*4, kernel_size=1, stride=stride, bias=False),\n",
    "            nn.BatchNorm2d(inner_channels*4)\n",
    "        )\n",
    "        layers = [ResidualBlock(input_channels, inner_channels, self.reduction_ratio, stride, projection)]\n",
    "        for layer in range(n_blocks-1):\n",
    "            layers.append(ResidualBlock(inner_channels * 4, inner_channels, self.reduction_ratio))\n",
    "        \n",
    "        return nn.Sequential(*layers)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.maxpool(x)\n",
    "        x = self.layers1(x)\n",
    "        x = self.layers2(x)\n",
    "        x = self.layers3(x)\n",
    "        x = self.layers4(x)\n",
    "        x = self.avgpool(x)\n",
    "        x = self.classifier(x)\n",
    "        return x"
   ],
   "id": "8f6ffb7b9ada31d4",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T10:10:13.523052Z",
     "start_time": "2024-10-13T10:10:13.186491Z"
    }
   },
   "cell_type": "code",
   "source": [
    "se_resnet_model = SEResNet(3, 1000, 16)\n",
    "summary(se_resnet_model, input_size=(1, 3, 224, 224), col_names=['input_size', 'output_size', 'mult_adds'], depth=4, device='cpu')"
   ],
   "id": "81c19b8e00414744",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "========================================================================================================================\n",
       "Layer (type:depth-idx)                        Input Shape               Output Shape              Mult-Adds\n",
       "========================================================================================================================\n",
       "SEResNet                                      [1, 3, 224, 224]          [1, 1000]                 --\n",
       "├─Sequential: 1-1                             [1, 3, 224, 224]          [1, 64, 112, 112]         --\n",
       "│    └─Conv2d: 2-1                            [1, 3, 224, 224]          [1, 64, 112, 112]         118,013,952\n",
       "│    └─BatchNorm2d: 2-2                       [1, 64, 112, 112]         [1, 64, 112, 112]         128\n",
       "│    └─ReLU: 2-3                              [1, 64, 112, 112]         [1, 64, 112, 112]         --\n",
       "├─MaxPool2d: 1-2                              [1, 64, 112, 112]         [1, 64, 56, 56]           --\n",
       "├─Sequential: 1-3                             [1, 64, 56, 56]           [1, 256, 56, 56]          --\n",
       "│    └─ResidualBlock: 2-4                     [1, 64, 56, 56]           [1, 256, 56, 56]          --\n",
       "│    │    └─Sequential: 3-1                   [1, 64, 56, 56]           [1, 256, 56, 56]          --\n",
       "│    │    │    └─Conv2d: 4-1                  [1, 64, 56, 56]           [1, 64, 56, 56]           12,845,056\n",
       "│    │    │    └─BatchNorm2d: 4-2             [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    │    └─ReLU: 4-3                    [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    │    └─Conv2d: 4-4                  [1, 64, 56, 56]           [1, 64, 56, 56]           115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-5             [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    │    └─ReLU: 4-6                    [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    │    └─Conv2d: 4-7                  [1, 64, 56, 56]           [1, 256, 56, 56]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-8             [1, 256, 56, 56]          [1, 256, 56, 56]          512\n",
       "│    │    └─SEBlock: 3-2                      [1, 256, 56, 56]          [256]                     --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-9       [1, 256, 56, 56]          [1, 256, 1, 1]            --\n",
       "│    │    │    └─Linear: 4-10                 [256]                     [16]                      65,536\n",
       "│    │    │    └─ReLU: 4-11                   [16]                      [16]                      --\n",
       "│    │    │    └─Linear: 4-12                 [16]                      [256]                     1,048,576\n",
       "│    │    │    └─Sigmoid: 4-13                [256]                     [256]                     --\n",
       "│    │    └─Sequential: 3-3                   [1, 64, 56, 56]           [1, 256, 56, 56]          --\n",
       "│    │    │    └─Conv2d: 4-14                 [1, 64, 56, 56]           [1, 256, 56, 56]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-15            [1, 256, 56, 56]          [1, 256, 56, 56]          512\n",
       "│    │    └─ReLU: 3-4                         [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    └─ResidualBlock: 2-5                     [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    │    └─Sequential: 3-5                   [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    │    │    └─Conv2d: 4-16                 [1, 256, 56, 56]          [1, 64, 56, 56]           51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-17            [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    │    └─ReLU: 4-18                   [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    │    └─Conv2d: 4-19                 [1, 64, 56, 56]           [1, 64, 56, 56]           115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-20            [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    │    └─ReLU: 4-21                   [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    │    └─Conv2d: 4-22                 [1, 64, 56, 56]           [1, 256, 56, 56]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-23            [1, 256, 56, 56]          [1, 256, 56, 56]          512\n",
       "│    │    └─SEBlock: 3-6                      [1, 256, 56, 56]          [256]                     --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-24      [1, 256, 56, 56]          [1, 256, 1, 1]            --\n",
       "│    │    │    └─Linear: 4-25                 [256]                     [16]                      65,536\n",
       "│    │    │    └─ReLU: 4-26                   [16]                      [16]                      --\n",
       "│    │    │    └─Linear: 4-27                 [16]                      [256]                     1,048,576\n",
       "│    │    │    └─Sigmoid: 4-28                [256]                     [256]                     --\n",
       "│    │    └─ReLU: 3-7                         [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    └─ResidualBlock: 2-6                     [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    │    └─Sequential: 3-8                   [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    │    │    └─Conv2d: 4-29                 [1, 256, 56, 56]          [1, 64, 56, 56]           51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-30            [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    │    └─ReLU: 4-31                   [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    │    └─Conv2d: 4-32                 [1, 64, 56, 56]           [1, 64, 56, 56]           115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-33            [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    │    └─ReLU: 4-34                   [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    │    └─Conv2d: 4-35                 [1, 64, 56, 56]           [1, 256, 56, 56]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-36            [1, 256, 56, 56]          [1, 256, 56, 56]          512\n",
       "│    │    └─SEBlock: 3-9                      [1, 256, 56, 56]          [256]                     --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-37      [1, 256, 56, 56]          [1, 256, 1, 1]            --\n",
       "│    │    │    └─Linear: 4-38                 [256]                     [16]                      65,536\n",
       "│    │    │    └─ReLU: 4-39                   [16]                      [16]                      --\n",
       "│    │    │    └─Linear: 4-40                 [16]                      [256]                     1,048,576\n",
       "│    │    │    └─Sigmoid: 4-41                [256]                     [256]                     --\n",
       "│    │    └─ReLU: 3-10                        [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "├─Sequential: 1-4                             [1, 256, 56, 56]          [1, 512, 28, 28]          --\n",
       "│    └─ResidualBlock: 2-7                     [1, 256, 56, 56]          [1, 512, 28, 28]          --\n",
       "│    │    └─Sequential: 3-11                  [1, 256, 56, 56]          [1, 512, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-42                 [1, 256, 56, 56]          [1, 128, 56, 56]          102,760,448\n",
       "│    │    │    └─BatchNorm2d: 4-43            [1, 128, 56, 56]          [1, 128, 56, 56]          256\n",
       "│    │    │    └─ReLU: 4-44                   [1, 128, 56, 56]          [1, 128, 56, 56]          --\n",
       "│    │    │    └─Conv2d: 4-45                 [1, 128, 56, 56]          [1, 128, 28, 28]          115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-46            [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    │    └─ReLU: 4-47                   [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-48                 [1, 128, 28, 28]          [1, 512, 28, 28]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-49            [1, 512, 28, 28]          [1, 512, 28, 28]          1,024\n",
       "│    │    └─SEBlock: 3-12                     [1, 512, 28, 28]          [512]                     --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-50      [1, 512, 28, 28]          [1, 512, 1, 1]            --\n",
       "│    │    │    └─Linear: 4-51                 [512]                     [32]                      524,288\n",
       "│    │    │    └─ReLU: 4-52                   [32]                      [32]                      --\n",
       "│    │    │    └─Linear: 4-53                 [32]                      [512]                     8,388,608\n",
       "│    │    │    └─Sigmoid: 4-54                [512]                     [512]                     --\n",
       "│    │    └─Sequential: 3-13                  [1, 256, 56, 56]          [1, 512, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-55                 [1, 256, 56, 56]          [1, 512, 28, 28]          102,760,448\n",
       "│    │    │    └─BatchNorm2d: 4-56            [1, 512, 28, 28]          [1, 512, 28, 28]          1,024\n",
       "│    │    └─ReLU: 3-14                        [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    └─ResidualBlock: 2-8                     [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    └─Sequential: 3-15                  [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-57                 [1, 512, 28, 28]          [1, 128, 28, 28]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-58            [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    │    └─ReLU: 4-59                   [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-60                 [1, 128, 28, 28]          [1, 128, 28, 28]          115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-61            [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    │    └─ReLU: 4-62                   [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-63                 [1, 128, 28, 28]          [1, 512, 28, 28]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-64            [1, 512, 28, 28]          [1, 512, 28, 28]          1,024\n",
       "│    │    └─SEBlock: 3-16                     [1, 512, 28, 28]          [512]                     --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-65      [1, 512, 28, 28]          [1, 512, 1, 1]            --\n",
       "│    │    │    └─Linear: 4-66                 [512]                     [32]                      524,288\n",
       "│    │    │    └─ReLU: 4-67                   [32]                      [32]                      --\n",
       "│    │    │    └─Linear: 4-68                 [32]                      [512]                     8,388,608\n",
       "│    │    │    └─Sigmoid: 4-69                [512]                     [512]                     --\n",
       "│    │    └─ReLU: 3-17                        [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    └─ResidualBlock: 2-9                     [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    └─Sequential: 3-18                  [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-70                 [1, 512, 28, 28]          [1, 128, 28, 28]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-71            [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    │    └─ReLU: 4-72                   [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-73                 [1, 128, 28, 28]          [1, 128, 28, 28]          115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-74            [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    │    └─ReLU: 4-75                   [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-76                 [1, 128, 28, 28]          [1, 512, 28, 28]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-77            [1, 512, 28, 28]          [1, 512, 28, 28]          1,024\n",
       "│    │    └─SEBlock: 3-19                     [1, 512, 28, 28]          [512]                     --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-78      [1, 512, 28, 28]          [1, 512, 1, 1]            --\n",
       "│    │    │    └─Linear: 4-79                 [512]                     [32]                      524,288\n",
       "│    │    │    └─ReLU: 4-80                   [32]                      [32]                      --\n",
       "│    │    │    └─Linear: 4-81                 [32]                      [512]                     8,388,608\n",
       "│    │    │    └─Sigmoid: 4-82                [512]                     [512]                     --\n",
       "│    │    └─ReLU: 3-20                        [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    └─ResidualBlock: 2-10                    [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    └─Sequential: 3-21                  [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-83                 [1, 512, 28, 28]          [1, 128, 28, 28]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-84            [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    │    └─ReLU: 4-85                   [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-86                 [1, 128, 28, 28]          [1, 128, 28, 28]          115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-87            [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    │    └─ReLU: 4-88                   [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-89                 [1, 128, 28, 28]          [1, 512, 28, 28]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-90            [1, 512, 28, 28]          [1, 512, 28, 28]          1,024\n",
       "│    │    └─SEBlock: 3-22                     [1, 512, 28, 28]          [512]                     --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-91      [1, 512, 28, 28]          [1, 512, 1, 1]            --\n",
       "│    │    │    └─Linear: 4-92                 [512]                     [32]                      524,288\n",
       "│    │    │    └─ReLU: 4-93                   [32]                      [32]                      --\n",
       "│    │    │    └─Linear: 4-94                 [32]                      [512]                     8,388,608\n",
       "│    │    │    └─Sigmoid: 4-95                [512]                     [512]                     --\n",
       "│    │    └─ReLU: 3-23                        [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "├─Sequential: 1-5                             [1, 512, 28, 28]          [1, 1024, 14, 14]         --\n",
       "│    └─ResidualBlock: 2-11                    [1, 512, 28, 28]          [1, 1024, 14, 14]         --\n",
       "│    │    └─Sequential: 3-24                  [1, 512, 28, 28]          [1, 1024, 14, 14]         --\n",
       "│    │    │    └─Conv2d: 4-96                 [1, 512, 28, 28]          [1, 256, 28, 28]          102,760,448\n",
       "│    │    │    └─BatchNorm2d: 4-97            [1, 256, 28, 28]          [1, 256, 28, 28]          512\n",
       "│    │    │    └─ReLU: 4-98                   [1, 256, 28, 28]          [1, 256, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-99                 [1, 256, 28, 28]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-100           [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    │    └─ReLU: 4-101                  [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-102                [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-103           [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SEBlock: 3-25                     [1, 1024, 14, 14]         [1024]                    --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-104     [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Linear: 4-105                [1024]                    [64]                      4,194,304\n",
       "│    │    │    └─ReLU: 4-106                  [64]                      [64]                      --\n",
       "│    │    │    └─Linear: 4-107                [64]                      [1024]                    67,108,864\n",
       "│    │    │    └─Sigmoid: 4-108               [1024]                    [1024]                    --\n",
       "│    │    └─Sequential: 3-26                  [1, 512, 28, 28]          [1, 1024, 14, 14]         --\n",
       "│    │    │    └─Conv2d: 4-109                [1, 512, 28, 28]          [1, 1024, 14, 14]         102,760,448\n",
       "│    │    │    └─BatchNorm2d: 4-110           [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─ReLU: 3-27                        [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    └─ResidualBlock: 2-12                    [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    └─Sequential: 3-28                  [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    │    └─Conv2d: 4-111                [1, 1024, 14, 14]         [1, 256, 14, 14]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-112           [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    │    └─ReLU: 4-113                  [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-114                [1, 256, 14, 14]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-115           [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    │    └─ReLU: 4-116                  [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-117                [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-118           [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SEBlock: 3-29                     [1, 1024, 14, 14]         [1024]                    --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-119     [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Linear: 4-120                [1024]                    [64]                      4,194,304\n",
       "│    │    │    └─ReLU: 4-121                  [64]                      [64]                      --\n",
       "│    │    │    └─Linear: 4-122                [64]                      [1024]                    67,108,864\n",
       "│    │    │    └─Sigmoid: 4-123               [1024]                    [1024]                    --\n",
       "│    │    └─ReLU: 3-30                        [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    └─ResidualBlock: 2-13                    [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    └─Sequential: 3-31                  [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    │    └─Conv2d: 4-124                [1, 1024, 14, 14]         [1, 256, 14, 14]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-125           [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    │    └─ReLU: 4-126                  [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-127                [1, 256, 14, 14]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-128           [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    │    └─ReLU: 4-129                  [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-130                [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-131           [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SEBlock: 3-32                     [1, 1024, 14, 14]         [1024]                    --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-132     [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Linear: 4-133                [1024]                    [64]                      4,194,304\n",
       "│    │    │    └─ReLU: 4-134                  [64]                      [64]                      --\n",
       "│    │    │    └─Linear: 4-135                [64]                      [1024]                    67,108,864\n",
       "│    │    │    └─Sigmoid: 4-136               [1024]                    [1024]                    --\n",
       "│    │    └─ReLU: 3-33                        [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    └─ResidualBlock: 2-14                    [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    └─Sequential: 3-34                  [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    │    └─Conv2d: 4-137                [1, 1024, 14, 14]         [1, 256, 14, 14]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-138           [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    │    └─ReLU: 4-139                  [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-140                [1, 256, 14, 14]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-141           [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    │    └─ReLU: 4-142                  [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-143                [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-144           [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SEBlock: 3-35                     [1, 1024, 14, 14]         [1024]                    --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-145     [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Linear: 4-146                [1024]                    [64]                      4,194,304\n",
       "│    │    │    └─ReLU: 4-147                  [64]                      [64]                      --\n",
       "│    │    │    └─Linear: 4-148                [64]                      [1024]                    67,108,864\n",
       "│    │    │    └─Sigmoid: 4-149               [1024]                    [1024]                    --\n",
       "│    │    └─ReLU: 3-36                        [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    └─ResidualBlock: 2-15                    [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    └─Sequential: 3-37                  [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    │    └─Conv2d: 4-150                [1, 1024, 14, 14]         [1, 256, 14, 14]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-151           [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    │    └─ReLU: 4-152                  [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-153                [1, 256, 14, 14]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-154           [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    │    └─ReLU: 4-155                  [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-156                [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-157           [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SEBlock: 3-38                     [1, 1024, 14, 14]         [1024]                    --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-158     [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Linear: 4-159                [1024]                    [64]                      4,194,304\n",
       "│    │    │    └─ReLU: 4-160                  [64]                      [64]                      --\n",
       "│    │    │    └─Linear: 4-161                [64]                      [1024]                    67,108,864\n",
       "│    │    │    └─Sigmoid: 4-162               [1024]                    [1024]                    --\n",
       "│    │    └─ReLU: 3-39                        [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    └─ResidualBlock: 2-16                    [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    └─Sequential: 3-40                  [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    │    └─Conv2d: 4-163                [1, 1024, 14, 14]         [1, 256, 14, 14]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-164           [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    │    └─ReLU: 4-165                  [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-166                [1, 256, 14, 14]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-167           [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    │    └─ReLU: 4-168                  [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-169                [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-170           [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SEBlock: 3-41                     [1, 1024, 14, 14]         [1024]                    --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-171     [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Linear: 4-172                [1024]                    [64]                      4,194,304\n",
       "│    │    │    └─ReLU: 4-173                  [64]                      [64]                      --\n",
       "│    │    │    └─Linear: 4-174                [64]                      [1024]                    67,108,864\n",
       "│    │    │    └─Sigmoid: 4-175               [1024]                    [1024]                    --\n",
       "│    │    └─ReLU: 3-42                        [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "├─Sequential: 1-6                             [1, 1024, 14, 14]         [1, 2048, 7, 7]           --\n",
       "│    └─ResidualBlock: 2-17                    [1, 1024, 14, 14]         [1, 2048, 7, 7]           --\n",
       "│    │    └─Sequential: 3-43                  [1, 1024, 14, 14]         [1, 2048, 7, 7]           --\n",
       "│    │    │    └─Conv2d: 4-176                [1, 1024, 14, 14]         [1, 512, 14, 14]          102,760,448\n",
       "│    │    │    └─BatchNorm2d: 4-177           [1, 512, 14, 14]          [1, 512, 14, 14]          1,024\n",
       "│    │    │    └─ReLU: 4-178                  [1, 512, 14, 14]          [1, 512, 14, 14]          --\n",
       "│    │    │    └─Conv2d: 4-179                [1, 512, 14, 14]          [1, 512, 7, 7]            115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-180           [1, 512, 7, 7]            [1, 512, 7, 7]            1,024\n",
       "│    │    │    └─ReLU: 4-181                  [1, 512, 7, 7]            [1, 512, 7, 7]            --\n",
       "│    │    │    └─Conv2d: 4-182                [1, 512, 7, 7]            [1, 2048, 7, 7]           51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-183           [1, 2048, 7, 7]           [1, 2048, 7, 7]           4,096\n",
       "│    │    └─SEBlock: 3-44                     [1, 2048, 7, 7]           [2048]                    --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-184     [1, 2048, 7, 7]           [1, 2048, 1, 1]           --\n",
       "│    │    │    └─Linear: 4-185                [2048]                    [128]                     33,554,432\n",
       "│    │    │    └─ReLU: 4-186                  [128]                     [128]                     --\n",
       "│    │    │    └─Linear: 4-187                [128]                     [2048]                    536,870,912\n",
       "│    │    │    └─Sigmoid: 4-188               [2048]                    [2048]                    --\n",
       "│    │    └─Sequential: 3-45                  [1, 1024, 14, 14]         [1, 2048, 7, 7]           --\n",
       "│    │    │    └─Conv2d: 4-189                [1, 1024, 14, 14]         [1, 2048, 7, 7]           102,760,448\n",
       "│    │    │    └─BatchNorm2d: 4-190           [1, 2048, 7, 7]           [1, 2048, 7, 7]           4,096\n",
       "│    │    └─ReLU: 3-46                        [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    └─ResidualBlock: 2-18                    [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    │    └─Sequential: 3-47                  [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    │    │    └─Conv2d: 4-191                [1, 2048, 7, 7]           [1, 512, 7, 7]            51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-192           [1, 512, 7, 7]            [1, 512, 7, 7]            1,024\n",
       "│    │    │    └─ReLU: 4-193                  [1, 512, 7, 7]            [1, 512, 7, 7]            --\n",
       "│    │    │    └─Conv2d: 4-194                [1, 512, 7, 7]            [1, 512, 7, 7]            115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-195           [1, 512, 7, 7]            [1, 512, 7, 7]            1,024\n",
       "│    │    │    └─ReLU: 4-196                  [1, 512, 7, 7]            [1, 512, 7, 7]            --\n",
       "│    │    │    └─Conv2d: 4-197                [1, 512, 7, 7]            [1, 2048, 7, 7]           51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-198           [1, 2048, 7, 7]           [1, 2048, 7, 7]           4,096\n",
       "│    │    └─SEBlock: 3-48                     [1, 2048, 7, 7]           [2048]                    --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-199     [1, 2048, 7, 7]           [1, 2048, 1, 1]           --\n",
       "│    │    │    └─Linear: 4-200                [2048]                    [128]                     33,554,432\n",
       "│    │    │    └─ReLU: 4-201                  [128]                     [128]                     --\n",
       "│    │    │    └─Linear: 4-202                [128]                     [2048]                    536,870,912\n",
       "│    │    │    └─Sigmoid: 4-203               [2048]                    [2048]                    --\n",
       "│    │    └─ReLU: 3-49                        [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    └─ResidualBlock: 2-19                    [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    │    └─Sequential: 3-50                  [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    │    │    └─Conv2d: 4-204                [1, 2048, 7, 7]           [1, 512, 7, 7]            51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-205           [1, 512, 7, 7]            [1, 512, 7, 7]            1,024\n",
       "│    │    │    └─ReLU: 4-206                  [1, 512, 7, 7]            [1, 512, 7, 7]            --\n",
       "│    │    │    └─Conv2d: 4-207                [1, 512, 7, 7]            [1, 512, 7, 7]            115,605,504\n",
       "│    │    │    └─BatchNorm2d: 4-208           [1, 512, 7, 7]            [1, 512, 7, 7]            1,024\n",
       "│    │    │    └─ReLU: 4-209                  [1, 512, 7, 7]            [1, 512, 7, 7]            --\n",
       "│    │    │    └─Conv2d: 4-210                [1, 512, 7, 7]            [1, 2048, 7, 7]           51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-211           [1, 2048, 7, 7]           [1, 2048, 7, 7]           4,096\n",
       "│    │    └─SEBlock: 3-51                     [1, 2048, 7, 7]           [2048]                    --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-212     [1, 2048, 7, 7]           [1, 2048, 1, 1]           --\n",
       "│    │    │    └─Linear: 4-213                [2048]                    [128]                     33,554,432\n",
       "│    │    │    └─ReLU: 4-214                  [128]                     [128]                     --\n",
       "│    │    │    └─Linear: 4-215                [128]                     [2048]                    536,870,912\n",
       "│    │    │    └─Sigmoid: 4-216               [2048]                    [2048]                    --\n",
       "│    │    └─ReLU: 3-52                        [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "├─AdaptiveAvgPool2d: 1-7                      [1, 2048, 7, 7]           [1, 2048, 1, 1]           --\n",
       "├─Sequential: 1-8                             [1, 2048, 1, 1]           [1, 1000]                 --\n",
       "│    └─Flatten: 2-20                          [1, 2048, 1, 1]           [1, 2048]                 --\n",
       "│    └─Linear: 2-21                           [1, 2048]                 [1, 1000]                 2,049,000\n",
       "========================================================================================================================\n",
       "Total params: 28,071,976\n",
       "Trainable params: 28,071,976\n",
       "Non-trainable params: 0\n",
       "Total mult-adds (Units.GIGABYTES): 6.27\n",
       "========================================================================================================================\n",
       "Input size (MB): 0.60\n",
       "Forward/backward pass size (MB): 177.96\n",
       "Params size (MB): 112.29\n",
       "Estimated Total Size (MB): 290.85\n",
       "========================================================================================================================"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T10:10:14.736879Z",
     "start_time": "2024-10-13T10:10:13.534463Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch.hub\n",
    "se_resnet_torch_model = torch.hub.load('moskomule/senet.pytorch', 'se_resnet50', num_classes=1000)\n",
    "summary(se_resnet_torch_model, input_size=(1, 3, 224, 224), col_names=['input_size', 'output_size', 'mult_adds'], depth=5, device='cpu')"
   ],
   "id": "db1cb73d8c7629f1",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in C:\\Users\\dk/.cache\\torch\\hub\\moskomule_senet.pytorch_master\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "========================================================================================================================\n",
       "Layer (type:depth-idx)                        Input Shape               Output Shape              Mult-Adds\n",
       "========================================================================================================================\n",
       "ResNet                                        [1, 3, 224, 224]          [1, 1000]                 --\n",
       "├─Conv2d: 1-1                                 [1, 3, 224, 224]          [1, 64, 112, 112]         118,013,952\n",
       "├─BatchNorm2d: 1-2                            [1, 64, 112, 112]         [1, 64, 112, 112]         128\n",
       "├─ReLU: 1-3                                   [1, 64, 112, 112]         [1, 64, 112, 112]         --\n",
       "├─MaxPool2d: 1-4                              [1, 64, 112, 112]         [1, 64, 56, 56]           --\n",
       "├─Sequential: 1-5                             [1, 64, 56, 56]           [1, 256, 56, 56]          --\n",
       "│    └─SEBottleneck: 2-1                      [1, 64, 56, 56]           [1, 256, 56, 56]          --\n",
       "│    │    └─Conv2d: 3-1                       [1, 64, 56, 56]           [1, 64, 56, 56]           12,845,056\n",
       "│    │    └─BatchNorm2d: 3-2                  [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    └─ReLU: 3-3                         [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    └─Conv2d: 3-4                       [1, 64, 56, 56]           [1, 64, 56, 56]           115,605,504\n",
       "│    │    └─BatchNorm2d: 3-5                  [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    └─ReLU: 3-6                         [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    └─Conv2d: 3-7                       [1, 64, 56, 56]           [1, 256, 56, 56]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-8                  [1, 256, 56, 56]          [1, 256, 56, 56]          512\n",
       "│    │    └─SELayer: 3-9                      [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-1       [1, 256, 56, 56]          [1, 256, 1, 1]            --\n",
       "│    │    │    └─Sequential: 4-2              [1, 256]                  [1, 256]                  --\n",
       "│    │    │    │    └─Linear: 5-1             [1, 256]                  [1, 16]                   4,096\n",
       "│    │    │    │    └─ReLU: 5-2               [1, 16]                   [1, 16]                   --\n",
       "│    │    │    │    └─Linear: 5-3             [1, 16]                   [1, 256]                  4,096\n",
       "│    │    │    │    └─Sigmoid: 5-4            [1, 256]                  [1, 256]                  --\n",
       "│    │    └─Sequential: 3-10                  [1, 64, 56, 56]           [1, 256, 56, 56]          --\n",
       "│    │    │    └─Conv2d: 4-3                  [1, 64, 56, 56]           [1, 256, 56, 56]          51,380,224\n",
       "│    │    │    └─BatchNorm2d: 4-4             [1, 256, 56, 56]          [1, 256, 56, 56]          512\n",
       "│    │    └─ReLU: 3-11                        [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    └─SEBottleneck: 2-2                      [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    │    └─Conv2d: 3-12                      [1, 256, 56, 56]          [1, 64, 56, 56]           51,380,224\n",
       "│    │    └─BatchNorm2d: 3-13                 [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    └─ReLU: 3-14                        [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    └─Conv2d: 3-15                      [1, 64, 56, 56]           [1, 64, 56, 56]           115,605,504\n",
       "│    │    └─BatchNorm2d: 3-16                 [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    └─ReLU: 3-17                        [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    └─Conv2d: 3-18                      [1, 64, 56, 56]           [1, 256, 56, 56]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-19                 [1, 256, 56, 56]          [1, 256, 56, 56]          512\n",
       "│    │    └─SELayer: 3-20                     [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-5       [1, 256, 56, 56]          [1, 256, 1, 1]            --\n",
       "│    │    │    └─Sequential: 4-6              [1, 256]                  [1, 256]                  --\n",
       "│    │    │    │    └─Linear: 5-5             [1, 256]                  [1, 16]                   4,096\n",
       "│    │    │    │    └─ReLU: 5-6               [1, 16]                   [1, 16]                   --\n",
       "│    │    │    │    └─Linear: 5-7             [1, 16]                   [1, 256]                  4,096\n",
       "│    │    │    │    └─Sigmoid: 5-8            [1, 256]                  [1, 256]                  --\n",
       "│    │    └─ReLU: 3-21                        [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    └─SEBottleneck: 2-3                      [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    │    └─Conv2d: 3-22                      [1, 256, 56, 56]          [1, 64, 56, 56]           51,380,224\n",
       "│    │    └─BatchNorm2d: 3-23                 [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    └─ReLU: 3-24                        [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    └─Conv2d: 3-25                      [1, 64, 56, 56]           [1, 64, 56, 56]           115,605,504\n",
       "│    │    └─BatchNorm2d: 3-26                 [1, 64, 56, 56]           [1, 64, 56, 56]           128\n",
       "│    │    └─ReLU: 3-27                        [1, 64, 56, 56]           [1, 64, 56, 56]           --\n",
       "│    │    └─Conv2d: 3-28                      [1, 64, 56, 56]           [1, 256, 56, 56]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-29                 [1, 256, 56, 56]          [1, 256, 56, 56]          512\n",
       "│    │    └─SELayer: 3-30                     [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-7       [1, 256, 56, 56]          [1, 256, 1, 1]            --\n",
       "│    │    │    └─Sequential: 4-8              [1, 256]                  [1, 256]                  --\n",
       "│    │    │    │    └─Linear: 5-9             [1, 256]                  [1, 16]                   4,096\n",
       "│    │    │    │    └─ReLU: 5-10              [1, 16]                   [1, 16]                   --\n",
       "│    │    │    │    └─Linear: 5-11            [1, 16]                   [1, 256]                  4,096\n",
       "│    │    │    │    └─Sigmoid: 5-12           [1, 256]                  [1, 256]                  --\n",
       "│    │    └─ReLU: 3-31                        [1, 256, 56, 56]          [1, 256, 56, 56]          --\n",
       "├─Sequential: 1-6                             [1, 256, 56, 56]          [1, 512, 28, 28]          --\n",
       "│    └─SEBottleneck: 2-4                      [1, 256, 56, 56]          [1, 512, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-32                      [1, 256, 56, 56]          [1, 128, 56, 56]          102,760,448\n",
       "│    │    └─BatchNorm2d: 3-33                 [1, 128, 56, 56]          [1, 128, 56, 56]          256\n",
       "│    │    └─ReLU: 3-34                        [1, 128, 56, 56]          [1, 128, 56, 56]          --\n",
       "│    │    └─Conv2d: 3-35                      [1, 128, 56, 56]          [1, 128, 28, 28]          115,605,504\n",
       "│    │    └─BatchNorm2d: 3-36                 [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    └─ReLU: 3-37                        [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-38                      [1, 128, 28, 28]          [1, 512, 28, 28]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-39                 [1, 512, 28, 28]          [1, 512, 28, 28]          1,024\n",
       "│    │    └─SELayer: 3-40                     [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-9       [1, 512, 28, 28]          [1, 512, 1, 1]            --\n",
       "│    │    │    └─Sequential: 4-10             [1, 512]                  [1, 512]                  --\n",
       "│    │    │    │    └─Linear: 5-13            [1, 512]                  [1, 32]                   16,384\n",
       "│    │    │    │    └─ReLU: 5-14              [1, 32]                   [1, 32]                   --\n",
       "│    │    │    │    └─Linear: 5-15            [1, 32]                   [1, 512]                  16,384\n",
       "│    │    │    │    └─Sigmoid: 5-16           [1, 512]                  [1, 512]                  --\n",
       "│    │    └─Sequential: 3-41                  [1, 256, 56, 56]          [1, 512, 28, 28]          --\n",
       "│    │    │    └─Conv2d: 4-11                 [1, 256, 56, 56]          [1, 512, 28, 28]          102,760,448\n",
       "│    │    │    └─BatchNorm2d: 4-12            [1, 512, 28, 28]          [1, 512, 28, 28]          1,024\n",
       "│    │    └─ReLU: 3-42                        [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    └─SEBottleneck: 2-5                      [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-43                      [1, 512, 28, 28]          [1, 128, 28, 28]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-44                 [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    └─ReLU: 3-45                        [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-46                      [1, 128, 28, 28]          [1, 128, 28, 28]          115,605,504\n",
       "│    │    └─BatchNorm2d: 3-47                 [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    └─ReLU: 3-48                        [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-49                      [1, 128, 28, 28]          [1, 512, 28, 28]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-50                 [1, 512, 28, 28]          [1, 512, 28, 28]          1,024\n",
       "│    │    └─SELayer: 3-51                     [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-13      [1, 512, 28, 28]          [1, 512, 1, 1]            --\n",
       "│    │    │    └─Sequential: 4-14             [1, 512]                  [1, 512]                  --\n",
       "│    │    │    │    └─Linear: 5-17            [1, 512]                  [1, 32]                   16,384\n",
       "│    │    │    │    └─ReLU: 5-18              [1, 32]                   [1, 32]                   --\n",
       "│    │    │    │    └─Linear: 5-19            [1, 32]                   [1, 512]                  16,384\n",
       "│    │    │    │    └─Sigmoid: 5-20           [1, 512]                  [1, 512]                  --\n",
       "│    │    └─ReLU: 3-52                        [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    └─SEBottleneck: 2-6                      [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-53                      [1, 512, 28, 28]          [1, 128, 28, 28]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-54                 [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    └─ReLU: 3-55                        [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-56                      [1, 128, 28, 28]          [1, 128, 28, 28]          115,605,504\n",
       "│    │    └─BatchNorm2d: 3-57                 [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    └─ReLU: 3-58                        [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-59                      [1, 128, 28, 28]          [1, 512, 28, 28]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-60                 [1, 512, 28, 28]          [1, 512, 28, 28]          1,024\n",
       "│    │    └─SELayer: 3-61                     [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-15      [1, 512, 28, 28]          [1, 512, 1, 1]            --\n",
       "│    │    │    └─Sequential: 4-16             [1, 512]                  [1, 512]                  --\n",
       "│    │    │    │    └─Linear: 5-21            [1, 512]                  [1, 32]                   16,384\n",
       "│    │    │    │    └─ReLU: 5-22              [1, 32]                   [1, 32]                   --\n",
       "│    │    │    │    └─Linear: 5-23            [1, 32]                   [1, 512]                  16,384\n",
       "│    │    │    │    └─Sigmoid: 5-24           [1, 512]                  [1, 512]                  --\n",
       "│    │    └─ReLU: 3-62                        [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    └─SEBottleneck: 2-7                      [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-63                      [1, 512, 28, 28]          [1, 128, 28, 28]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-64                 [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    └─ReLU: 3-65                        [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-66                      [1, 128, 28, 28]          [1, 128, 28, 28]          115,605,504\n",
       "│    │    └─BatchNorm2d: 3-67                 [1, 128, 28, 28]          [1, 128, 28, 28]          256\n",
       "│    │    └─ReLU: 3-68                        [1, 128, 28, 28]          [1, 128, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-69                      [1, 128, 28, 28]          [1, 512, 28, 28]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-70                 [1, 512, 28, 28]          [1, 512, 28, 28]          1,024\n",
       "│    │    └─SELayer: 3-71                     [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-17      [1, 512, 28, 28]          [1, 512, 1, 1]            --\n",
       "│    │    │    └─Sequential: 4-18             [1, 512]                  [1, 512]                  --\n",
       "│    │    │    │    └─Linear: 5-25            [1, 512]                  [1, 32]                   16,384\n",
       "│    │    │    │    └─ReLU: 5-26              [1, 32]                   [1, 32]                   --\n",
       "│    │    │    │    └─Linear: 5-27            [1, 32]                   [1, 512]                  16,384\n",
       "│    │    │    │    └─Sigmoid: 5-28           [1, 512]                  [1, 512]                  --\n",
       "│    │    └─ReLU: 3-72                        [1, 512, 28, 28]          [1, 512, 28, 28]          --\n",
       "├─Sequential: 1-7                             [1, 512, 28, 28]          [1, 1024, 14, 14]         --\n",
       "│    └─SEBottleneck: 2-8                      [1, 512, 28, 28]          [1, 1024, 14, 14]         --\n",
       "│    │    └─Conv2d: 3-73                      [1, 512, 28, 28]          [1, 256, 28, 28]          102,760,448\n",
       "│    │    └─BatchNorm2d: 3-74                 [1, 256, 28, 28]          [1, 256, 28, 28]          512\n",
       "│    │    └─ReLU: 3-75                        [1, 256, 28, 28]          [1, 256, 28, 28]          --\n",
       "│    │    └─Conv2d: 3-76                      [1, 256, 28, 28]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    └─BatchNorm2d: 3-77                 [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    └─ReLU: 3-78                        [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-79                      [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    └─BatchNorm2d: 3-80                 [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SELayer: 3-81                     [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-19      [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Sequential: 4-20             [1, 1024]                 [1, 1024]                 --\n",
       "│    │    │    │    └─Linear: 5-29            [1, 1024]                 [1, 64]                   65,536\n",
       "│    │    │    │    └─ReLU: 5-30              [1, 64]                   [1, 64]                   --\n",
       "│    │    │    │    └─Linear: 5-31            [1, 64]                   [1, 1024]                 65,536\n",
       "│    │    │    │    └─Sigmoid: 5-32           [1, 1024]                 [1, 1024]                 --\n",
       "│    │    └─Sequential: 3-82                  [1, 512, 28, 28]          [1, 1024, 14, 14]         --\n",
       "│    │    │    └─Conv2d: 4-21                 [1, 512, 28, 28]          [1, 1024, 14, 14]         102,760,448\n",
       "│    │    │    └─BatchNorm2d: 4-22            [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─ReLU: 3-83                        [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    └─SEBottleneck: 2-9                      [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    └─Conv2d: 3-84                      [1, 1024, 14, 14]         [1, 256, 14, 14]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-85                 [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    └─ReLU: 3-86                        [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-87                      [1, 256, 14, 14]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    └─BatchNorm2d: 3-88                 [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    └─ReLU: 3-89                        [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-90                      [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    └─BatchNorm2d: 3-91                 [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SELayer: 3-92                     [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-23      [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Sequential: 4-24             [1, 1024]                 [1, 1024]                 --\n",
       "│    │    │    │    └─Linear: 5-33            [1, 1024]                 [1, 64]                   65,536\n",
       "│    │    │    │    └─ReLU: 5-34              [1, 64]                   [1, 64]                   --\n",
       "│    │    │    │    └─Linear: 5-35            [1, 64]                   [1, 1024]                 65,536\n",
       "│    │    │    │    └─Sigmoid: 5-36           [1, 1024]                 [1, 1024]                 --\n",
       "│    │    └─ReLU: 3-93                        [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    └─SEBottleneck: 2-10                     [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    └─Conv2d: 3-94                      [1, 1024, 14, 14]         [1, 256, 14, 14]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-95                 [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    └─ReLU: 3-96                        [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-97                      [1, 256, 14, 14]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    └─BatchNorm2d: 3-98                 [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    └─ReLU: 3-99                        [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-100                     [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    └─BatchNorm2d: 3-101                [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SELayer: 3-102                    [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-25      [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Sequential: 4-26             [1, 1024]                 [1, 1024]                 --\n",
       "│    │    │    │    └─Linear: 5-37            [1, 1024]                 [1, 64]                   65,536\n",
       "│    │    │    │    └─ReLU: 5-38              [1, 64]                   [1, 64]                   --\n",
       "│    │    │    │    └─Linear: 5-39            [1, 64]                   [1, 1024]                 65,536\n",
       "│    │    │    │    └─Sigmoid: 5-40           [1, 1024]                 [1, 1024]                 --\n",
       "│    │    └─ReLU: 3-103                       [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    └─SEBottleneck: 2-11                     [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    └─Conv2d: 3-104                     [1, 1024, 14, 14]         [1, 256, 14, 14]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-105                [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    └─ReLU: 3-106                       [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-107                     [1, 256, 14, 14]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    └─BatchNorm2d: 3-108                [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    └─ReLU: 3-109                       [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-110                     [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    └─BatchNorm2d: 3-111                [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SELayer: 3-112                    [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-27      [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Sequential: 4-28             [1, 1024]                 [1, 1024]                 --\n",
       "│    │    │    │    └─Linear: 5-41            [1, 1024]                 [1, 64]                   65,536\n",
       "│    │    │    │    └─ReLU: 5-42              [1, 64]                   [1, 64]                   --\n",
       "│    │    │    │    └─Linear: 5-43            [1, 64]                   [1, 1024]                 65,536\n",
       "│    │    │    │    └─Sigmoid: 5-44           [1, 1024]                 [1, 1024]                 --\n",
       "│    │    └─ReLU: 3-113                       [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    └─SEBottleneck: 2-12                     [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    └─Conv2d: 3-114                     [1, 1024, 14, 14]         [1, 256, 14, 14]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-115                [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    └─ReLU: 3-116                       [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-117                     [1, 256, 14, 14]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    └─BatchNorm2d: 3-118                [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    └─ReLU: 3-119                       [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-120                     [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    └─BatchNorm2d: 3-121                [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SELayer: 3-122                    [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-29      [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Sequential: 4-30             [1, 1024]                 [1, 1024]                 --\n",
       "│    │    │    │    └─Linear: 5-45            [1, 1024]                 [1, 64]                   65,536\n",
       "│    │    │    │    └─ReLU: 5-46              [1, 64]                   [1, 64]                   --\n",
       "│    │    │    │    └─Linear: 5-47            [1, 64]                   [1, 1024]                 65,536\n",
       "│    │    │    │    └─Sigmoid: 5-48           [1, 1024]                 [1, 1024]                 --\n",
       "│    │    └─ReLU: 3-123                       [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    └─SEBottleneck: 2-13                     [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    └─Conv2d: 3-124                     [1, 1024, 14, 14]         [1, 256, 14, 14]          51,380,224\n",
       "│    │    └─BatchNorm2d: 3-125                [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    └─ReLU: 3-126                       [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-127                     [1, 256, 14, 14]          [1, 256, 14, 14]          115,605,504\n",
       "│    │    └─BatchNorm2d: 3-128                [1, 256, 14, 14]          [1, 256, 14, 14]          512\n",
       "│    │    └─ReLU: 3-129                       [1, 256, 14, 14]          [1, 256, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-130                     [1, 256, 14, 14]          [1, 1024, 14, 14]         51,380,224\n",
       "│    │    └─BatchNorm2d: 3-131                [1, 1024, 14, 14]         [1, 1024, 14, 14]         2,048\n",
       "│    │    └─SELayer: 3-132                    [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-31      [1, 1024, 14, 14]         [1, 1024, 1, 1]           --\n",
       "│    │    │    └─Sequential: 4-32             [1, 1024]                 [1, 1024]                 --\n",
       "│    │    │    │    └─Linear: 5-49            [1, 1024]                 [1, 64]                   65,536\n",
       "│    │    │    │    └─ReLU: 5-50              [1, 64]                   [1, 64]                   --\n",
       "│    │    │    │    └─Linear: 5-51            [1, 64]                   [1, 1024]                 65,536\n",
       "│    │    │    │    └─Sigmoid: 5-52           [1, 1024]                 [1, 1024]                 --\n",
       "│    │    └─ReLU: 3-133                       [1, 1024, 14, 14]         [1, 1024, 14, 14]         --\n",
       "├─Sequential: 1-8                             [1, 1024, 14, 14]         [1, 2048, 7, 7]           --\n",
       "│    └─SEBottleneck: 2-14                     [1, 1024, 14, 14]         [1, 2048, 7, 7]           --\n",
       "│    │    └─Conv2d: 3-134                     [1, 1024, 14, 14]         [1, 512, 14, 14]          102,760,448\n",
       "│    │    └─BatchNorm2d: 3-135                [1, 512, 14, 14]          [1, 512, 14, 14]          1,024\n",
       "│    │    └─ReLU: 3-136                       [1, 512, 14, 14]          [1, 512, 14, 14]          --\n",
       "│    │    └─Conv2d: 3-137                     [1, 512, 14, 14]          [1, 512, 7, 7]            115,605,504\n",
       "│    │    └─BatchNorm2d: 3-138                [1, 512, 7, 7]            [1, 512, 7, 7]            1,024\n",
       "│    │    └─ReLU: 3-139                       [1, 512, 7, 7]            [1, 512, 7, 7]            --\n",
       "│    │    └─Conv2d: 3-140                     [1, 512, 7, 7]            [1, 2048, 7, 7]           51,380,224\n",
       "│    │    └─BatchNorm2d: 3-141                [1, 2048, 7, 7]           [1, 2048, 7, 7]           4,096\n",
       "│    │    └─SELayer: 3-142                    [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-33      [1, 2048, 7, 7]           [1, 2048, 1, 1]           --\n",
       "│    │    │    └─Sequential: 4-34             [1, 2048]                 [1, 2048]                 --\n",
       "│    │    │    │    └─Linear: 5-53            [1, 2048]                 [1, 128]                  262,144\n",
       "│    │    │    │    └─ReLU: 5-54              [1, 128]                  [1, 128]                  --\n",
       "│    │    │    │    └─Linear: 5-55            [1, 128]                  [1, 2048]                 262,144\n",
       "│    │    │    │    └─Sigmoid: 5-56           [1, 2048]                 [1, 2048]                 --\n",
       "│    │    └─Sequential: 3-143                 [1, 1024, 14, 14]         [1, 2048, 7, 7]           --\n",
       "│    │    │    └─Conv2d: 4-35                 [1, 1024, 14, 14]         [1, 2048, 7, 7]           102,760,448\n",
       "│    │    │    └─BatchNorm2d: 4-36            [1, 2048, 7, 7]           [1, 2048, 7, 7]           4,096\n",
       "│    │    └─ReLU: 3-144                       [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    └─SEBottleneck: 2-15                     [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    │    └─Conv2d: 3-145                     [1, 2048, 7, 7]           [1, 512, 7, 7]            51,380,224\n",
       "│    │    └─BatchNorm2d: 3-146                [1, 512, 7, 7]            [1, 512, 7, 7]            1,024\n",
       "│    │    └─ReLU: 3-147                       [1, 512, 7, 7]            [1, 512, 7, 7]            --\n",
       "│    │    └─Conv2d: 3-148                     [1, 512, 7, 7]            [1, 512, 7, 7]            115,605,504\n",
       "│    │    └─BatchNorm2d: 3-149                [1, 512, 7, 7]            [1, 512, 7, 7]            1,024\n",
       "│    │    └─ReLU: 3-150                       [1, 512, 7, 7]            [1, 512, 7, 7]            --\n",
       "│    │    └─Conv2d: 3-151                     [1, 512, 7, 7]            [1, 2048, 7, 7]           51,380,224\n",
       "│    │    └─BatchNorm2d: 3-152                [1, 2048, 7, 7]           [1, 2048, 7, 7]           4,096\n",
       "│    │    └─SELayer: 3-153                    [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-37      [1, 2048, 7, 7]           [1, 2048, 1, 1]           --\n",
       "│    │    │    └─Sequential: 4-38             [1, 2048]                 [1, 2048]                 --\n",
       "│    │    │    │    └─Linear: 5-57            [1, 2048]                 [1, 128]                  262,144\n",
       "│    │    │    │    └─ReLU: 5-58              [1, 128]                  [1, 128]                  --\n",
       "│    │    │    │    └─Linear: 5-59            [1, 128]                  [1, 2048]                 262,144\n",
       "│    │    │    │    └─Sigmoid: 5-60           [1, 2048]                 [1, 2048]                 --\n",
       "│    │    └─ReLU: 3-154                       [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    └─SEBottleneck: 2-16                     [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    │    └─Conv2d: 3-155                     [1, 2048, 7, 7]           [1, 512, 7, 7]            51,380,224\n",
       "│    │    └─BatchNorm2d: 3-156                [1, 512, 7, 7]            [1, 512, 7, 7]            1,024\n",
       "│    │    └─ReLU: 3-157                       [1, 512, 7, 7]            [1, 512, 7, 7]            --\n",
       "│    │    └─Conv2d: 3-158                     [1, 512, 7, 7]            [1, 512, 7, 7]            115,605,504\n",
       "│    │    └─BatchNorm2d: 3-159                [1, 512, 7, 7]            [1, 512, 7, 7]            1,024\n",
       "│    │    └─ReLU: 3-160                       [1, 512, 7, 7]            [1, 512, 7, 7]            --\n",
       "│    │    └─Conv2d: 3-161                     [1, 512, 7, 7]            [1, 2048, 7, 7]           51,380,224\n",
       "│    │    └─BatchNorm2d: 3-162                [1, 2048, 7, 7]           [1, 2048, 7, 7]           4,096\n",
       "│    │    └─SELayer: 3-163                    [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "│    │    │    └─AdaptiveAvgPool2d: 4-39      [1, 2048, 7, 7]           [1, 2048, 1, 1]           --\n",
       "│    │    │    └─Sequential: 4-40             [1, 2048]                 [1, 2048]                 --\n",
       "│    │    │    │    └─Linear: 5-61            [1, 2048]                 [1, 128]                  262,144\n",
       "│    │    │    │    └─ReLU: 5-62              [1, 128]                  [1, 128]                  --\n",
       "│    │    │    │    └─Linear: 5-63            [1, 128]                  [1, 2048]                 262,144\n",
       "│    │    │    │    └─Sigmoid: 5-64           [1, 2048]                 [1, 2048]                 --\n",
       "│    │    └─ReLU: 3-164                       [1, 2048, 7, 7]           [1, 2048, 7, 7]           --\n",
       "├─AdaptiveAvgPool2d: 1-9                      [1, 2048, 7, 7]           [1, 2048, 1, 1]           --\n",
       "├─Linear: 1-10                                [1, 2048]                 [1, 1000]                 2,049,000\n",
       "========================================================================================================================\n",
       "Total params: 28,071,976\n",
       "Trainable params: 28,071,976\n",
       "Non-trainable params: 0\n",
       "Total mult-adds (Units.GIGABYTES): 4.09\n",
       "========================================================================================================================\n",
       "Input size (MB): 0.60\n",
       "Forward/backward pass size (MB): 177.96\n",
       "Params size (MB): 112.29\n",
       "Estimated Total Size (MB): 290.85\n",
       "========================================================================================================================"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## Training",
   "id": "e2f0ce17f284d70c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T10:10:16.310378Z",
     "start_time": "2024-10-13T10:10:14.812438Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from pathlib import Path\n",
    "\n",
    "TRAIN_RATIO = 0.8\n",
    "data_dir = Path('./data/')\n",
    "\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize(224),\n",
    "    transforms.ToTensor(),\n",
    "])\n",
    "\n",
    "train_ds = datasets.CIFAR100(data_dir, train=True, download=True, transform=transform)\n",
    "train_ds, val_ds = random_split(train_ds, (TRAIN_RATIO, 1 - TRAIN_RATIO))\n",
    "val_ds.transform = transform\n",
    "test_ds = datasets.CIFAR100(data_dir, train=False, download=True, transform=transform)"
   ],
   "id": "42d21d784414347d",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-13T12:09:55.308915Z",
     "start_time": "2024-10-13T10:10:16.318326Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import wandb\n",
    "from src.engine import *\n",
    "\n",
    "config = dict(batch_size=64, lr=5e-4, epochs=20, dataset='CIFAR100')\n",
    "with wandb.init(project='pytorch-study', name='SEResNet50', config=config) as run:\n",
    "    w_config = run.config\n",
    "    train_dl = DataLoader(train_ds, batch_size=w_config.batch_size, shuffle=True)\n",
    "    val_dl = DataLoader(val_ds, batch_size=w_config.batch_size, shuffle=True)\n",
    "    \n",
    "    n_classes = len(train_ds.dataset.classes)\n",
    "    se_resnet_mdoel = SEResNet(3, n_classes, 16).to(DEVICE)\n",
    "        \n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.Adam(se_resnet_mdoel.parameters(), lr=w_config.lr)\n",
    "    \n",
    "    loss_history, acc_history = train(se_resnet_mdoel, train_dl, val_dl, criterion, optimizer, w_config.epochs, DEVICE, run) "
   ],
   "id": "7feb6758495dd8a0",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch=20: 100%|██████████| 20/20 [1:59:31<00:00, 358.56s/it, train_loss=0.143, train_acc=95.50%, val_loss=2.801, val_acc=51.36%]  \n"
     ]
    }
   ],
   "execution_count": 7
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}