AdNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torchsummary import summary\n",
    "import torch.nn as nn\n",
    "from collections import OrderedDict\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import DataLoader\n",
    "device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(model_choice, channels=64):\n",
    "    #load resnet18\n",
    "    if model_choice==1:\n",
    "        # Build a resnet50 model from scratch\n",
    "        model = torchvision.models.vgg13(pretrained=True)\n",
    "        model.classifier[6] = nn.Linear(4096, 10)\n",
    "        \n",
    "\n",
    "    else:\n",
    "        model=nn.Sequential(OrderedDict([\n",
    "            (\"conv1\",nn.Conv2d(3,channels,7,3,3,bias=False)),\n",
    "            (\"bn1\",nn.BatchNorm2d(channels)),\n",
    "            (\"relu1\",nn.ReLU(inplace=True)),\n",
    "            (\"maxpool1\",nn.MaxPool2d(2,2,1)),\n",
    "            \n",
    "            (\"conv2\",nn.Conv2d(channels,channels,3,1,1,bias=False)),\n",
    "            (\"bn2\",nn.BatchNorm2d(channels)),\n",
    "            (\"relu2\",nn.ReLU(inplace=True)),\n",
    "            (\"maxpool2\",nn.MaxPool2d(2,2,1)),\n",
    "\n",
    "            (\"conv3\",nn.Conv2d(channels,channels,3,1,1,bias=False)),\n",
    "            (\"bn3\",nn.BatchNorm2d(channels)),\n",
    "            (\"relu3\",nn.ReLU(inplace=True)),\n",
    "            (\"maxpool3\",nn.MaxPool2d(2,2,1)),\n",
    "            (\"conv4\",nn.Conv2d(channels,channels,3,1,1,bias=False)),\n",
    "            (\"bn4\",nn.BatchNorm2d(channels)),\n",
    "            (\"relu4\",nn.ReLU(inplace=True)),\n",
    "            (\"maxpool4\",nn.MaxPool2d(2,2,1)),\n",
    "            (\"conv5\",nn.Conv2d(channels,channels,3,1,1,bias=False)),\n",
    "            (\"bn5\",nn.BatchNorm2d(channels)),\n",
    "            (\"relu5\",nn.ReLU(inplace=True)),\n",
    "            (\"maxpool5\",nn.MaxPool2d(2,2,1)),\n",
    "            (\"conv6\",nn.Conv2d(channels,channels,3,1,1,bias=False)),\n",
    "            (\"bn6\",nn.BatchNorm2d(channels)),\n",
    "            (\"relu6\",nn.ReLU(inplace=True)), \n",
    "            (\"maxpool6\",nn.MaxPool2d(2,2,1)),\n",
    "            (\"conv7\",nn.Conv2d(channels,channels,3,1,1,bias=False)),\n",
    "            (\"bn7\",nn.BatchNorm2d(channels)),\n",
    "            (\"relu7\",nn.ReLU(inplace=True)), \n",
    "            (\"maxpool7\",nn.MaxPool2d(2,2,1)),\n",
    "            (\"fl\",nn.Flatten()),\n",
    "            (\"fc\",nn.Linear(4*channels,10)),\n",
    "            (\"softmax\",nn.Softmax(dim=1))\n",
    "        ]))\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "#model=model.to(device)\n",
    "\n",
    "cnt = 0\n",
    "conv_dict = dict()\n",
    "def hook_fn(module, input, output):\n",
    "    global cnt\n",
    "    conv_dict[conv_list[cnt]] = module.weight\n",
    "    cnt += 1\n",
    "    return None\n",
    "\n",
    "\n",
    "def get_hook(model):\n",
    "    dic=model.state_dict()\n",
    "    bn_layers = dict()\n",
    "    conv_dict = dict()\n",
    "    conv_list = []\n",
    "    flag=False\n",
    "    layer=None\n",
    "    hook=[]\n",
    "    for keys, values in dic.items():\n",
    "        if 'conv' in keys:\n",
    "            name=keys.replace('.weight','')\n",
    "            layer=model.get_submodule(name)\n",
    "            for i in range(100):\n",
    "                name=name.replace(\".\"+str(i)+\".\",\"[\"+str(i)+\"].\")\n",
    "            layer.name = name\n",
    "            hook.append(layer.register_forward_hook(hook_fn))\n",
    "            conv_list.append(name)\n",
    "            flag=True\n",
    "        elif flag:\n",
    "            if 'bn' in keys:\n",
    "                bn_layers[name]=keys.replace('.weight','')\n",
    "                for i in range(100):\n",
    "                    bn_layers[name]=bn_layers[name].replace(\".\"+str(i)+\".\",\"[\"+str(i)+\"].\")\n",
    "            else:\n",
    "                bn_layers[name]=None\n",
    "            flag=False\n",
    "    return hook, conv_list, bn_layers\n",
    "\n",
    "def get_variance(tensor):\n",
    "    var=torch.mean(torch.var(tensor, dim=(0)))\n",
    "    print(\"variance:\",var)\n",
    "    return var\n",
    "\n",
    "\n",
    "\n",
    "def add_channels(name, mean=0, std=0.01, add_coef=2, max_out_channels=512):\n",
    "    layer=eval(\"model.\"+name)\n",
    "    \n",
    "    if layer.out_channels*add_coef>max_out_channels:\n",
    "        print(\"\",name,\" out_channels reach max_out_channels:\",max_out_channels)\n",
    "        return\n",
    "    shape=layer.weight.shape\n",
    "    ori_weight=layer.weight.clone()\n",
    "\n",
    "    # Add output channels to conv\n",
    "    exec(\"{}=torch.nn.utils.skip_init(torch.nn.Conv2d,in_channels=layer.in_channels, out_channels=int(layer.out_channels*add_coef), kernel_size=layer.kernel_size, stride=layer.stride, padding=layer.padding, dilation=layer.dilation, groups=layer.groups, bias=layer.bias, padding_mode=layer.padding_mode, device=device)\".format(\"model.\"+name)) \n",
    "    add_layer_shape=list(ori_weight.shape)\n",
    "    add_layer_shape[0]=int(add_layer_shape[0]*(add_coef-1))\n",
    "    # Load original weight and add new weight\n",
    "    exec(\"{}.weight=torch.nn.Parameter(torch.cat([ori_weight,torch.normal(mean, std, size=add_layer_shape).to(device)], dim=0))\".format(\"model.\"+name))\n",
    "    # print(torch.cat([ori_weight,torch.normal(mean, std, size=add_layer_shape).to(device)], dim=0).shape)\n",
    "    print(\"add {} out_channels to layer:\".format((add_coef-1)*ori_weight.shape[0]),name,\"out_channels:\",layer.out_channels,\"->\",layer.out_channels*add_coef)\n",
    "\n",
    "    next = conv_list.index(name) + 1\n",
    "\n",
    "    # Add input channels to next conv layer\n",
    "    if next < len(conv_list):\n",
    "        name_next = conv_list[next]\n",
    "        layer = eval(\"model.\"+name_next)\n",
    "        weight = layer.weight.clone()\n",
    "        exec(\"{}=torch.nn.utils.skip_init(torch.nn.Conv2d,in_channels=int(layer.in_channels*add_coef), out_channels=layer.out_channels, kernel_size=layer.kernel_size, stride=layer.stride, padding=layer.padding, dilation=layer.dilation, groups=layer.groups, bias=layer.bias, padding_mode=layer.padding_mode, device=device)\".format(\"model.\"+name_next)) \n",
    "        add_layer_shape=list(weight.shape)\n",
    "        add_layer_shape[1]=int(add_layer_shape[1]*(add_coef-1))\n",
    "        # print(torch.cat((weight,torch.normal(mean,std,size=add_layer_shape).to(device)), dim=1).shape)\n",
    "        exec(\"{}.weight=torch.nn.Parameter(torch.cat((weight,torch.normal(mean,std,size=add_layer_shape).to(device)), dim=1))\".format(\"model.\"+name_next))\n",
    "    else:\n",
    "        weight = model.fc.weight.clone()\n",
    "        model.fc.in_features = int(model.fc.in_features * add_coef)\n",
    "        add_layer_shape=list(weight.shape)\n",
    "        add_layer_shape[1]=int(add_layer_shape[1]*(add_coef-1))\n",
    "        model.fc=nn.Linear(model.fc.in_features, model.fc.out_features,device=device)\n",
    "        model.fc.weight=torch.nn.Parameter(torch.cat((weight,torch.normal(mean,std,size=add_layer_shape).to(device)), dim=1))\n",
    "\n",
    "\n",
    "    # Add input channels to bn\n",
    "    if(bn_layers[name] != None):\n",
    "        bn=eval(\"model.\"+bn_layers[name])\n",
    "        bn_weight=bn.weight.clone()\n",
    "        add_layer_shape=list(bn_weight.shape)\n",
    "        add_layer_shape[0]=int(add_layer_shape[0]*(add_coef-1))\n",
    "        exec(\"{}=torch.nn.BatchNorm2d(num_features=int(bn.num_features*add_coef), eps=bn.eps, momentum=bn.momentum, affine=bn.affine, track_running_stats=bn.track_running_stats, device=device)\".format(\"model.\"+bn_layers[name]))\n",
    "        # Load original weight and add new weight\n",
    "        exec(\"{}.weight=torch.nn.Parameter(torch.cat((bn_weight,torch.normal(mean,std,size=add_layer_shape).to(device)), dim=0))\".format(\"model.\"+bn_layers[name]))\n",
    "    \n",
    "\n",
    "\n",
    "# Modify the layers(If variance > 0.5, then add the channels of this layer)\n",
    "def modify_layers(layers, var_threshold=0.2, mean=0, std=0.1, add_coef=2, max_out_channels=512):\n",
    "    for keys, output in layers.items():\n",
    "        var= get_variance(output)\n",
    "        #print(var)\n",
    "        if var>var_threshold:\n",
    "            flag=True\n",
    "            add_channels(keys, mean, std, add_coef, max_out_channels)\n",
    "    layers.clear()\n",
    "# model= create_model(0)\n",
    "# model= model.to(device)\n",
    "\n",
    "# hook, conv_list, bn_layers = get_hook(model)\n",
    "\n",
    "# model.train()\n",
    "# random_batch = torch.rand(1, 3, 224,224).to(device)\n",
    "# model(random_batch)\n",
    "\n",
    "# modify_layers(conv_dict, var_threshold=0.2)\n",
    "# for h in hook:\n",
    "#     h.remove()\n",
    "# #print(model)\n",
    "# model(random_batch)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# load cifar10\n",
    "#load the CIFAR-10 dataset, and resize the data into 224x224\n",
    "myTransforms = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.RandomHorizontalFlip(p=0.5),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])\n",
    "\n",
    "cifar10_data = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=myTransforms)\n",
    "cifar10_data_test = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=myTransforms)\n",
    "\n",
    "#load the data\n",
    "train_loader=DataLoader(cifar10_data,batch_size=64,shuffle=True, num_workers=0)\n",
    "test_loader=DataLoader(cifar10_data_test,batch_size=64,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_model(model):\n",
    "    total=0\n",
    "    correct=0\n",
    "    validation_accuracy=0\n",
    "    with torch.no_grad():\n",
    "        for data,target in test_loader:\n",
    "            data,target=data.to(device),target.to(device)\n",
    "            output=model.forward(data)\n",
    "            _,predicted=torch.max(output.data,1)\n",
    "            total+=target.size(0)\n",
    "            correct+=(predicted==target).sum().item()\n",
    "        validation_accuracy=100*correct/total\n",
    "        print(\"Accuracy of the network on the 10000 test images: {}%\".format(validation_accuracy),'\\n')\n",
    "    return validation_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normal_train(model, optimizer, loss_f, epochs, device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')):\n",
    "    model.train()\n",
    "    training_loss=0.0\n",
    "    for epoch in range(epochs):\n",
    "        for batch_idx, (data, target) in enumerate(train_loader):\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            output = model.forward(data)\n",
    "            loss = loss_f(output, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            training_loss+=loss.item()\n",
    "            if batch_idx % 10 == 0 :\n",
    "                print('[iteration - %3d] training loss: %.3f' % (epoch*len(train_loader) + batch_idx, training_loss/10))\n",
    "                training_loss = 0.0\n",
    "                print()\n",
    "        del data, target, output\n",
    "        torch.cuda.empty_cache()\n",
    "                \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Adnn_train(model, optimizer, loss_f, epochs, duration=100, var_threshold=0.3, mean=0, std=0.01, add_coef=2, max_out_channels=512, validation=True, patience=5 , device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')):\n",
    "    global conv_list, bn_layers, cnt\n",
    "    best_acc=0\n",
    "    best_epoch=0\n",
    "    for epoch in range(epochs):\n",
    "        model.train()\n",
    "        for batch_idx, (data, target) in enumerate(train_loader):\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            output = model.forward(data)\n",
    "            loss = loss_f(output, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            print('\\r[Eposh - %3d][iteration - %3d / %3d ] training loss: %.3f' % (epoch+1, batch_idx, len(train_loader), loss.item()), end=\"\")\n",
    "            if batch_idx%duration==duration-1:\n",
    "                print()\n",
    "                cnt=0\n",
    "                # Get a batch from the training set\n",
    "                hook, conv_list, bn_layers = get_hook(model)\n",
    "                model.forward(data)\n",
    "                # Modify the layers(If variance > 0.5, then add the channels of this layer)\n",
    "                modify_layers(conv_dict, var_threshold=var_threshold, mean=mean, std=std, add_coef=add_coef, max_out_channels=max_out_channels)\n",
    "                del optimizer\n",
    "                optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.9)\n",
    "                for h in hook:\n",
    "                    h.remove()\n",
    "                del hook, conv_list, bn_layers\n",
    "                torch.cuda.empty_cache()\n",
    "        print()\n",
    "        if validation:\n",
    "                model.eval()\n",
    "                acc=test_model(model)\n",
    "                if acc>best_acc:\n",
    "                    best_acc=acc\n",
    "                    best_epoch=epoch\n",
    "                model.train()\n",
    "                if epoch-best_epoch>patience:\n",
    "                    print(\"Early stopping\")\n",
    "                    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Eposh -   1][iteration -  99 / 782 ] training loss: 2.269\n",
      "tensor(0.0023, device='cuda:0', grad_fn=<MeanBackward0>)\n",
      "tensor(0.0012, device='cuda:0', grad_fn=<MeanBackward0>)\n",
      "tensor(0.0012, device='cuda:0', grad_fn=<MeanBackward0>)\n",
      "tensor(0.0012, device='cuda:0', grad_fn=<MeanBackward0>)\n",
      "tensor(0.0011, device='cuda:0', grad_fn=<MeanBackward0>)\n",
      "tensor(0.0012, device='cuda:0', grad_fn=<MeanBackward0>)\n",
      "tensor(0.0012, device='cuda:0', grad_fn=<MeanBackward0>)\n",
      "[Eposh -   1][iteration - 135 / 782 ] training loss: 2.267"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[23], line 6\u001b[0m\n\u001b[0;32m      4\u001b[0m loss_f \u001b[39m=\u001b[39m nn\u001b[39m.\u001b[39mCrossEntropyLoss()\n\u001b[0;32m      5\u001b[0m Epochs\u001b[39m=\u001b[39m\u001b[39m50\u001b[39m\n\u001b[1;32m----> 6\u001b[0m Adnn_train(model, optimizer, loss_f, Epochs, var_threshold\u001b[39m=\u001b[39;49m\u001b[39m0.15\u001b[39;49m, max_out_channels\u001b[39m=\u001b[39;49m\u001b[39m1024\u001b[39;49m, add_coef\u001b[39m=\u001b[39;49m\u001b[39m1.5\u001b[39;49m)\n\u001b[0;32m      7\u001b[0m test_model(model)\n\u001b[0;32m      8\u001b[0m summary(model,input_size\u001b[39m=\u001b[39m(\u001b[39m3\u001b[39m,\u001b[39m224\u001b[39m,\u001b[39m224\u001b[39m))\n",
      "Cell \u001b[1;32mIn[22], line 8\u001b[0m, in \u001b[0;36mAdnn_train\u001b[1;34m(model, optimizer, loss_f, epochs, duration, var_threshold, mean, std, add_coef, max_out_channels, validation, patience, device)\u001b[0m\n\u001b[0;32m      6\u001b[0m model\u001b[39m.\u001b[39mtrain()\n\u001b[0;32m      7\u001b[0m \u001b[39mfor\u001b[39;00m batch_idx, (data, target) \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(train_loader):\n\u001b[1;32m----> 8\u001b[0m     data, target \u001b[39m=\u001b[39m data\u001b[39m.\u001b[39;49mto(device), target\u001b[39m.\u001b[39mto(device)\n\u001b[0;32m      9\u001b[0m     optimizer\u001b[39m.\u001b[39mzero_grad()\n\u001b[0;32m     10\u001b[0m     output \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39mforward(data)\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#Adnn train\n",
    "model=create_model(0,32).to(device)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.9)\n",
    "loss_f = nn.CrossEntropyLoss()\n",
    "Epochs=50\n",
    "Adnn_train(model, optimizer, loss_f, Epochs, var_threshold=0.15, max_out_channels=1024, add_coef=1.5)\n",
    "test_model(model)\n",
    "summary(model,input_size=(3,224,224))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[iteration -   0] training loss: 0.230\n",
      "\n",
      "[iteration -  10] training loss: 2.297\n",
      "\n",
      "[iteration -  20] training loss: 2.286\n",
      "\n",
      "[iteration -  30] training loss: 2.261\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model=create_model(0,256).to(device)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.9)\n",
    "loss_f = nn.CrossEntropyLoss()\n",
    "Epochs=10\n",
    "normal_train(model, optimizer, loss_f, Epochs)\n",
    "test_model(model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the network on the 10000 test images: 78.24% \n",
      "\n",
      "=================================================================\n",
      "Layer (type:depth-idx)                   Param #\n",
      "=================================================================\n",
      "├─Conv2d: 1-1                            37,632\n",
      "├─BatchNorm2d: 1-2                       512\n",
      "├─ReLU: 1-3                              --\n",
      "├─MaxPool2d: 1-4                         --\n",
      "├─Conv2d: 1-5                            589,824\n",
      "├─BatchNorm2d: 1-6                       512\n",
      "├─ReLU: 1-7                              --\n",
      "├─MaxPool2d: 1-8                         --\n",
      "├─Conv2d: 1-9                            589,824\n",
      "├─BatchNorm2d: 1-10                      512\n",
      "├─ReLU: 1-11                             --\n",
      "├─MaxPool2d: 1-12                        --\n",
      "├─Conv2d: 1-13                           589,824\n",
      "├─BatchNorm2d: 1-14                      512\n",
      "├─ReLU: 1-15                             --\n",
      "├─MaxPool2d: 1-16                        --\n",
      "├─Conv2d: 1-17                           589,824\n",
      "├─BatchNorm2d: 1-18                      512\n",
      "├─ReLU: 1-19                             --\n",
      "├─MaxPool2d: 1-20                        --\n",
      "├─Conv2d: 1-21                           589,824\n",
      "├─BatchNorm2d: 1-22                      512\n",
      "├─ReLU: 1-23                             --\n",
      "├─MaxPool2d: 1-24                        --\n",
      "├─Conv2d: 1-25                           589,824\n",
      "├─BatchNorm2d: 1-26                      512\n",
      "├─ReLU: 1-27                             --\n",
      "├─MaxPool2d: 1-28                        --\n",
      "├─Flatten: 1-29                          --\n",
      "├─Linear: 1-30                           10,250\n",
      "├─Softmax: 1-31                          --\n",
      "=================================================================\n",
      "Total params: 3,590,410\n",
      "Trainable params: 3,590,410\n",
      "Non-trainable params: 0\n",
      "=================================================================\n"
     ]
    }
   ],
   "source": [
    "test_model(model)\n",
    "summary(model,input_size=(3,224,224))\n",
    "torch.cuda.empty_cache()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[iteration -   0] training loss: 0.229\n",
      "\n",
      "[iteration -  10] training loss: 2.302\n",
      "\n",
      "[iteration -  20] training loss: 2.300\n",
      "\n",
      "[iteration -  30] training loss: 2.299\n",
      "\n",
      "[iteration -  40] training loss: 2.294\n",
      "\n",
      "[iteration -  50] training loss: 2.292\n",
      "\n",
      "[iteration -  60] training loss: 2.287\n",
      "\n",
      "[iteration -  70] training loss: 2.282\n",
      "\n",
      "[iteration -  80] training loss: 2.276\n",
      "\n",
      "[iteration -  90] training loss: 2.277\n",
      "\n",
      "[iteration - 100] training loss: 2.277\n",
      "\n",
      "[iteration - 110] training loss: 2.278\n",
      "\n",
      "[iteration - 120] training loss: 2.260\n",
      "\n",
      "[iteration - 130] training loss: 2.260\n",
      "\n",
      "[iteration - 140] training loss: 2.258\n",
      "\n",
      "[iteration - 150] training loss: 2.257\n",
      "\n",
      "[iteration - 160] training loss: 2.247\n",
      "\n",
      "[iteration - 170] training loss: 2.248\n",
      "\n",
      "[iteration - 180] training loss: 2.236\n",
      "\n",
      "[iteration - 190] training loss: 2.241\n",
      "\n",
      "[iteration - 200] training loss: 2.225\n",
      "\n",
      "[iteration - 210] training loss: 2.230\n",
      "\n",
      "[iteration - 220] training loss: 2.236\n",
      "\n",
      "[iteration - 230] training loss: 2.208\n",
      "\n",
      "[iteration - 240] training loss: 2.210\n",
      "\n",
      "[iteration - 250] training loss: 2.202\n",
      "\n",
      "[iteration - 260] training loss: 2.199\n",
      "\n",
      "[iteration - 270] training loss: 2.212\n",
      "\n",
      "[iteration - 280] training loss: 2.202\n",
      "\n",
      "[iteration - 290] training loss: 2.207\n",
      "\n",
      "[iteration - 300] training loss: 2.208\n",
      "\n",
      "[iteration - 310] training loss: 2.216\n",
      "\n",
      "[iteration - 320] training loss: 2.181\n",
      "\n",
      "[iteration - 330] training loss: 2.193\n",
      "\n",
      "[iteration - 340] training loss: 2.203\n",
      "\n",
      "[iteration - 350] training loss: 2.190\n",
      "\n",
      "[iteration - 360] training loss: 2.189\n",
      "\n",
      "[iteration - 370] training loss: 2.190\n",
      "\n",
      "[iteration - 380] training loss: 2.179\n",
      "\n",
      "[iteration - 390] training loss: 2.165\n",
      "\n",
      "[iteration - 400] training loss: 2.184\n",
      "\n",
      "[iteration - 410] training loss: 2.188\n",
      "\n",
      "[iteration - 420] training loss: 2.185\n",
      "\n",
      "[iteration - 430] training loss: 2.175\n",
      "\n",
      "[iteration - 440] training loss: 2.176\n",
      "\n",
      "[iteration - 450] training loss: 2.184\n",
      "\n",
      "[iteration - 460] training loss: 2.160\n",
      "\n",
      "[iteration - 470] training loss: 2.166\n",
      "\n",
      "[iteration - 480] training loss: 2.155\n",
      "\n",
      "[iteration - 490] training loss: 2.160\n",
      "\n",
      "[iteration - 500] training loss: 2.157\n",
      "\n",
      "[iteration - 510] training loss: 2.146\n",
      "\n",
      "[iteration - 520] training loss: 2.175\n",
      "\n",
      "[iteration - 530] training loss: 2.163\n",
      "\n",
      "[iteration - 540] training loss: 2.155\n",
      "\n",
      "[iteration - 550] training loss: 2.157\n",
      "\n",
      "[iteration - 560] training loss: 2.153\n",
      "\n",
      "[iteration - 570] training loss: 2.157\n",
      "\n",
      "[iteration - 580] training loss: 2.152\n",
      "\n",
      "[iteration - 590] training loss: 2.129\n",
      "\n",
      "[iteration - 600] training loss: 2.126\n",
      "\n",
      "[iteration - 610] training loss: 2.139\n",
      "\n",
      "[iteration - 620] training loss: 2.133\n",
      "\n",
      "[iteration - 630] training loss: 2.129\n",
      "\n",
      "[iteration - 640] training loss: 2.134\n",
      "\n",
      "[iteration - 650] training loss: 2.132\n",
      "\n",
      "[iteration - 660] training loss: 2.126\n",
      "\n",
      "[iteration - 670] training loss: 2.127\n",
      "\n",
      "[iteration - 680] training loss: 2.147\n",
      "\n",
      "[iteration - 690] training loss: 2.125\n",
      "\n",
      "[iteration - 700] training loss: 2.121\n",
      "\n",
      "[iteration - 710] training loss: 2.098\n",
      "\n",
      "[iteration - 720] training loss: 2.087\n",
      "\n",
      "[iteration - 730] training loss: 2.138\n",
      "\n",
      "[iteration - 740] training loss: 2.115\n",
      "\n",
      "[iteration - 750] training loss: 2.114\n",
      "\n",
      "[iteration - 760] training loss: 2.108\n",
      "\n",
      "[iteration - 770] training loss: 2.109\n",
      "\n",
      "[iteration - 780] training loss: 2.089\n",
      "\n",
      "[iteration - 782] training loss: 0.424\n",
      "\n",
      "[iteration - 792] training loss: 2.079\n",
      "\n",
      "[iteration - 802] training loss: 2.115\n",
      "\n",
      "[iteration - 812] training loss: 2.098\n",
      "\n",
      "[iteration - 822] training loss: 2.110\n",
      "\n",
      "[iteration - 832] training loss: 2.095\n",
      "\n",
      "[iteration - 842] training loss: 2.102\n",
      "\n",
      "[iteration - 852] training loss: 2.084\n",
      "\n",
      "[iteration - 862] training loss: 2.114\n",
      "\n",
      "[iteration - 872] training loss: 2.107\n",
      "\n",
      "[iteration - 882] training loss: 2.084\n",
      "\n",
      "[iteration - 892] training loss: 2.102\n",
      "\n",
      "[iteration - 902] training loss: 2.110\n",
      "\n",
      "[iteration - 912] training loss: 2.063\n",
      "\n",
      "[iteration - 922] training loss: 2.098\n",
      "\n",
      "[iteration - 932] training loss: 2.061\n",
      "\n",
      "[iteration - 942] training loss: 2.092\n",
      "\n",
      "[iteration - 952] training loss: 2.092\n",
      "\n",
      "[iteration - 962] training loss: 2.061\n",
      "\n",
      "[iteration - 972] training loss: 2.113\n",
      "\n",
      "[iteration - 982] training loss: 2.070\n",
      "\n",
      "[iteration - 992] training loss: 2.080\n",
      "\n",
      "[iteration - 1002] training loss: 2.064\n",
      "\n",
      "[iteration - 1012] training loss: 2.087\n",
      "\n",
      "[iteration - 1022] training loss: 2.078\n",
      "\n",
      "[iteration - 1032] training loss: 2.086\n",
      "\n",
      "[iteration - 1042] training loss: 2.063\n",
      "\n",
      "[iteration - 1052] training loss: 2.086\n",
      "\n",
      "[iteration - 1062] training loss: 2.073\n",
      "\n",
      "[iteration - 1072] training loss: 2.056\n",
      "\n",
      "[iteration - 1082] training loss: 2.065\n",
      "\n",
      "[iteration - 1092] training loss: 2.077\n",
      "\n",
      "[iteration - 1102] training loss: 2.081\n",
      "\n",
      "[iteration - 1112] training loss: 2.054\n",
      "\n",
      "[iteration - 1122] training loss: 2.075\n",
      "\n",
      "[iteration - 1132] training loss: 2.074\n",
      "\n",
      "[iteration - 1142] training loss: 2.061\n",
      "\n",
      "[iteration - 1152] training loss: 2.069\n",
      "\n",
      "[iteration - 1162] training loss: 2.051\n",
      "\n",
      "[iteration - 1172] training loss: 2.060\n",
      "\n",
      "[iteration - 1182] training loss: 2.085\n",
      "\n",
      "[iteration - 1192] training loss: 2.068\n",
      "\n",
      "[iteration - 1202] training loss: 2.050\n",
      "\n",
      "[iteration - 1212] training loss: 2.041\n",
      "\n",
      "[iteration - 1222] training loss: 2.053\n",
      "\n",
      "[iteration - 1232] training loss: 2.040\n",
      "\n",
      "[iteration - 1242] training loss: 2.053\n",
      "\n",
      "[iteration - 1252] training loss: 2.034\n",
      "\n",
      "[iteration - 1262] training loss: 2.054\n",
      "\n",
      "[iteration - 1272] training loss: 2.088\n",
      "\n",
      "[iteration - 1282] training loss: 2.078\n",
      "\n",
      "[iteration - 1292] training loss: 2.051\n",
      "\n",
      "[iteration - 1302] training loss: 2.071\n",
      "\n",
      "[iteration - 1312] training loss: 2.038\n",
      "\n",
      "[iteration - 1322] training loss: 2.054\n",
      "\n",
      "[iteration - 1332] training loss: 2.035\n",
      "\n",
      "[iteration - 1342] training loss: 2.054\n",
      "\n",
      "[iteration - 1352] training loss: 2.075\n",
      "\n",
      "[iteration - 1362] training loss: 2.038\n",
      "\n",
      "[iteration - 1372] training loss: 2.054\n",
      "\n",
      "[iteration - 1382] training loss: 2.032\n",
      "\n",
      "[iteration - 1392] training loss: 2.041\n",
      "\n",
      "[iteration - 1402] training loss: 2.059\n",
      "\n",
      "[iteration - 1412] training loss: 2.042\n",
      "\n",
      "[iteration - 1422] training loss: 2.045\n",
      "\n",
      "[iteration - 1432] training loss: 2.037\n",
      "\n",
      "[iteration - 1442] training loss: 2.067\n",
      "\n",
      "[iteration - 1452] training loss: 2.039\n",
      "\n",
      "[iteration - 1462] training loss: 2.032\n",
      "\n",
      "[iteration - 1472] training loss: 2.040\n",
      "\n",
      "[iteration - 1482] training loss: 2.060\n",
      "\n",
      "[iteration - 1492] training loss: 2.040\n",
      "\n",
      "[iteration - 1502] training loss: 2.047\n",
      "\n",
      "[iteration - 1512] training loss: 2.028\n",
      "\n",
      "[iteration - 1522] training loss: 2.027\n",
      "\n",
      "[iteration - 1532] training loss: 2.001\n",
      "\n",
      "[iteration - 1542] training loss: 2.021\n",
      "\n",
      "[iteration - 1552] training loss: 2.037\n",
      "\n",
      "[iteration - 1562] training loss: 2.048\n",
      "\n",
      "[iteration - 1564] training loss: 0.410\n",
      "\n",
      "[iteration - 1574] training loss: 2.025\n",
      "\n",
      "[iteration - 1584] training loss: 2.066\n",
      "\n",
      "[iteration - 1594] training loss: 2.028\n",
      "\n",
      "[iteration - 1604] training loss: 2.056\n",
      "\n",
      "[iteration - 1614] training loss: 2.013\n",
      "\n",
      "[iteration - 1624] training loss: 2.016\n",
      "\n",
      "[iteration - 1634] training loss: 1.993\n",
      "\n",
      "[iteration - 1644] training loss: 2.032\n",
      "\n",
      "[iteration - 1654] training loss: 2.044\n",
      "\n",
      "[iteration - 1664] training loss: 2.006\n",
      "\n",
      "[iteration - 1674] training loss: 2.023\n",
      "\n",
      "[iteration - 1684] training loss: 2.019\n",
      "\n",
      "[iteration - 1694] training loss: 2.016\n",
      "\n",
      "[iteration - 1704] training loss: 2.042\n",
      "\n",
      "[iteration - 1714] training loss: 2.022\n",
      "\n",
      "[iteration - 1724] training loss: 2.006\n",
      "\n",
      "[iteration - 1734] training loss: 2.015\n",
      "\n",
      "[iteration - 1744] training loss: 2.005\n",
      "\n",
      "[iteration - 1754] training loss: 2.012\n",
      "\n",
      "[iteration - 1764] training loss: 2.026\n",
      "\n",
      "[iteration - 1774] training loss: 2.024\n",
      "\n",
      "[iteration - 1784] training loss: 2.024\n",
      "\n",
      "[iteration - 1794] training loss: 2.018\n",
      "\n",
      "[iteration - 1804] training loss: 2.010\n",
      "\n",
      "[iteration - 1814] training loss: 2.009\n",
      "\n",
      "[iteration - 1824] training loss: 1.991\n",
      "\n",
      "[iteration - 1834] training loss: 1.999\n",
      "\n",
      "[iteration - 1844] training loss: 1.985\n",
      "\n",
      "[iteration - 1854] training loss: 2.006\n",
      "\n",
      "[iteration - 1864] training loss: 2.004\n",
      "\n",
      "[iteration - 1874] training loss: 2.022\n",
      "\n",
      "[iteration - 1884] training loss: 2.000\n",
      "\n",
      "[iteration - 1894] training loss: 2.041\n",
      "\n",
      "[iteration - 1904] training loss: 1.996\n",
      "\n",
      "[iteration - 1914] training loss: 1.986\n",
      "\n",
      "[iteration - 1924] training loss: 1.989\n",
      "\n",
      "[iteration - 1934] training loss: 2.000\n",
      "\n",
      "[iteration - 1944] training loss: 1.980\n",
      "\n",
      "[iteration - 1954] training loss: 2.014\n",
      "\n",
      "[iteration - 1964] training loss: 2.003\n",
      "\n",
      "[iteration - 1974] training loss: 2.003\n",
      "\n",
      "[iteration - 1984] training loss: 1.981\n",
      "\n",
      "[iteration - 1994] training loss: 1.987\n",
      "\n",
      "[iteration - 2004] training loss: 1.999\n",
      "\n",
      "[iteration - 2014] training loss: 1.982\n",
      "\n",
      "[iteration - 2024] training loss: 1.980\n",
      "\n",
      "[iteration - 2034] training loss: 2.010\n",
      "\n",
      "[iteration - 2044] training loss: 1.972\n",
      "\n",
      "[iteration - 2054] training loss: 2.004\n",
      "\n",
      "[iteration - 2064] training loss: 1.982\n",
      "\n",
      "[iteration - 2074] training loss: 2.005\n",
      "\n",
      "[iteration - 2084] training loss: 1.981\n",
      "\n",
      "[iteration - 2094] training loss: 1.984\n",
      "\n",
      "[iteration - 2104] training loss: 2.004\n",
      "\n",
      "[iteration - 2114] training loss: 1.968\n",
      "\n",
      "[iteration - 2124] training loss: 1.973\n",
      "\n",
      "[iteration - 2134] training loss: 1.962\n",
      "\n",
      "[iteration - 2144] training loss: 1.974\n",
      "\n",
      "[iteration - 2154] training loss: 2.000\n",
      "\n",
      "[iteration - 2164] training loss: 1.979\n",
      "\n",
      "[iteration - 2174] training loss: 1.950\n",
      "\n",
      "[iteration - 2184] training loss: 1.953\n",
      "\n",
      "[iteration - 2194] training loss: 1.959\n",
      "\n",
      "[iteration - 2204] training loss: 1.978\n",
      "\n",
      "[iteration - 2214] training loss: 1.971\n",
      "\n",
      "[iteration - 2224] training loss: 1.963\n",
      "\n",
      "[iteration - 2234] training loss: 1.975\n",
      "\n",
      "[iteration - 2244] training loss: 2.003\n",
      "\n",
      "[iteration - 2254] training loss: 1.983\n",
      "\n",
      "[iteration - 2264] training loss: 1.961\n",
      "\n",
      "[iteration - 2274] training loss: 1.973\n",
      "\n",
      "[iteration - 2284] training loss: 1.971\n",
      "\n",
      "[iteration - 2294] training loss: 2.007\n",
      "\n",
      "[iteration - 2304] training loss: 1.977\n",
      "\n",
      "[iteration - 2314] training loss: 2.004\n",
      "\n",
      "[iteration - 2324] training loss: 1.963\n",
      "\n",
      "[iteration - 2334] training loss: 1.985\n",
      "\n",
      "[iteration - 2344] training loss: 1.970\n",
      "\n",
      "[iteration - 2346] training loss: 0.396\n",
      "\n",
      "[iteration - 2356] training loss: 1.972\n",
      "\n",
      "[iteration - 2366] training loss: 1.968\n",
      "\n",
      "[iteration - 2376] training loss: 1.971\n",
      "\n",
      "[iteration - 2386] training loss: 1.961\n",
      "\n",
      "[iteration - 2396] training loss: 1.964\n",
      "\n",
      "[iteration - 2406] training loss: 1.983\n",
      "\n",
      "[iteration - 2416] training loss: 1.968\n",
      "\n",
      "[iteration - 2426] training loss: 1.968\n",
      "\n",
      "[iteration - 2436] training loss: 1.964\n",
      "\n",
      "[iteration - 2446] training loss: 1.955\n",
      "\n",
      "[iteration - 2456] training loss: 1.941\n",
      "\n",
      "[iteration - 2466] training loss: 1.953\n",
      "\n",
      "[iteration - 2476] training loss: 1.960\n",
      "\n",
      "[iteration - 2486] training loss: 1.931\n",
      "\n",
      "[iteration - 2496] training loss: 1.957\n",
      "\n",
      "[iteration - 2506] training loss: 1.972\n",
      "\n",
      "[iteration - 2516] training loss: 1.962\n",
      "\n",
      "[iteration - 2526] training loss: 1.961\n",
      "\n",
      "[iteration - 2536] training loss: 1.949\n",
      "\n",
      "[iteration - 2546] training loss: 1.942\n",
      "\n",
      "[iteration - 2556] training loss: 1.966\n",
      "\n",
      "[iteration - 2566] training loss: 1.979\n",
      "\n",
      "[iteration - 2576] training loss: 1.938\n",
      "\n",
      "[iteration - 2586] training loss: 1.920\n",
      "\n",
      "[iteration - 2596] training loss: 1.936\n",
      "\n",
      "[iteration - 2606] training loss: 1.944\n",
      "\n",
      "[iteration - 2616] training loss: 1.935\n",
      "\n",
      "[iteration - 2626] training loss: 1.937\n",
      "\n",
      "[iteration - 2636] training loss: 1.977\n",
      "\n",
      "[iteration - 2646] training loss: 1.938\n",
      "\n",
      "[iteration - 2656] training loss: 1.922\n",
      "\n",
      "[iteration - 2666] training loss: 1.962\n",
      "\n",
      "[iteration - 2676] training loss: 1.944\n",
      "\n",
      "[iteration - 2686] training loss: 1.963\n",
      "\n",
      "[iteration - 2696] training loss: 1.963\n",
      "\n",
      "[iteration - 2706] training loss: 1.940\n",
      "\n",
      "[iteration - 2716] training loss: 1.982\n",
      "\n",
      "[iteration - 2726] training loss: 1.936\n",
      "\n",
      "[iteration - 2736] training loss: 1.978\n",
      "\n",
      "[iteration - 2746] training loss: 1.934\n",
      "\n",
      "[iteration - 2756] training loss: 1.933\n",
      "\n",
      "[iteration - 2766] training loss: 1.962\n",
      "\n",
      "[iteration - 2776] training loss: 1.918\n",
      "\n",
      "[iteration - 2786] training loss: 1.940\n",
      "\n",
      "[iteration - 2796] training loss: 1.965\n",
      "\n",
      "[iteration - 2806] training loss: 1.965\n",
      "\n",
      "[iteration - 2816] training loss: 1.924\n",
      "\n",
      "[iteration - 2826] training loss: 1.934\n",
      "\n",
      "[iteration - 2836] training loss: 1.947\n",
      "\n",
      "[iteration - 2846] training loss: 1.914\n",
      "\n",
      "[iteration - 2856] training loss: 1.921\n",
      "\n",
      "[iteration - 2866] training loss: 1.941\n",
      "\n",
      "[iteration - 2876] training loss: 1.932\n",
      "\n",
      "[iteration - 2886] training loss: 1.913\n",
      "\n",
      "[iteration - 2896] training loss: 1.931\n",
      "\n",
      "[iteration - 2906] training loss: 1.939\n",
      "\n",
      "[iteration - 2916] training loss: 1.958\n",
      "\n",
      "[iteration - 2926] training loss: 1.947\n",
      "\n",
      "[iteration - 2936] training loss: 1.946\n",
      "\n",
      "[iteration - 2946] training loss: 1.949\n",
      "\n",
      "[iteration - 2956] training loss: 1.942\n",
      "\n",
      "[iteration - 2966] training loss: 1.925\n",
      "\n",
      "[iteration - 2976] training loss: 1.924\n",
      "\n",
      "[iteration - 2986] training loss: 1.935\n",
      "\n",
      "[iteration - 2996] training loss: 1.916\n",
      "\n",
      "[iteration - 3006] training loss: 1.925\n",
      "\n",
      "[iteration - 3016] training loss: 1.932\n",
      "\n",
      "[iteration - 3026] training loss: 1.924\n",
      "\n",
      "[iteration - 3036] training loss: 1.940\n",
      "\n",
      "[iteration - 3046] training loss: 1.955\n",
      "\n",
      "[iteration - 3056] training loss: 1.945\n",
      "\n",
      "[iteration - 3066] training loss: 1.908\n",
      "\n",
      "[iteration - 3076] training loss: 1.956\n",
      "\n",
      "[iteration - 3086] training loss: 1.941\n",
      "\n",
      "[iteration - 3096] training loss: 1.930\n",
      "\n",
      "[iteration - 3106] training loss: 1.910\n",
      "\n",
      "[iteration - 3116] training loss: 1.953\n",
      "\n",
      "[iteration - 3126] training loss: 1.927\n",
      "\n",
      "[iteration - 3128] training loss: 0.389\n",
      "\n",
      "[iteration - 3138] training loss: 1.923\n",
      "\n",
      "[iteration - 3148] training loss: 1.885\n",
      "\n",
      "[iteration - 3158] training loss: 1.927\n",
      "\n",
      "[iteration - 3168] training loss: 1.932\n",
      "\n",
      "[iteration - 3178] training loss: 1.907\n",
      "\n",
      "[iteration - 3188] training loss: 1.920\n",
      "\n",
      "[iteration - 3198] training loss: 1.940\n",
      "\n",
      "[iteration - 3208] training loss: 1.911\n",
      "\n",
      "[iteration - 3218] training loss: 1.931\n",
      "\n",
      "[iteration - 3228] training loss: 1.930\n",
      "\n",
      "[iteration - 3238] training loss: 1.909\n",
      "\n",
      "[iteration - 3248] training loss: 1.913\n",
      "\n",
      "[iteration - 3258] training loss: 1.928\n",
      "\n",
      "[iteration - 3268] training loss: 1.912\n",
      "\n",
      "[iteration - 3278] training loss: 1.930\n",
      "\n",
      "[iteration - 3288] training loss: 1.898\n",
      "\n",
      "[iteration - 3298] training loss: 1.925\n",
      "\n",
      "[iteration - 3308] training loss: 1.930\n",
      "\n",
      "[iteration - 3318] training loss: 1.918\n",
      "\n",
      "[iteration - 3328] training loss: 1.911\n",
      "\n",
      "[iteration - 3338] training loss: 1.896\n",
      "\n",
      "[iteration - 3348] training loss: 1.938\n",
      "\n",
      "[iteration - 3358] training loss: 1.913\n",
      "\n",
      "[iteration - 3368] training loss: 1.897\n",
      "\n",
      "[iteration - 3378] training loss: 1.931\n",
      "\n",
      "[iteration - 3388] training loss: 1.922\n",
      "\n",
      "[iteration - 3398] training loss: 1.886\n",
      "\n",
      "[iteration - 3408] training loss: 1.893\n",
      "\n",
      "[iteration - 3418] training loss: 1.917\n",
      "\n",
      "[iteration - 3428] training loss: 1.896\n",
      "\n",
      "[iteration - 3438] training loss: 1.892\n",
      "\n",
      "[iteration - 3448] training loss: 1.908\n",
      "\n",
      "[iteration - 3458] training loss: 1.924\n",
      "\n",
      "[iteration - 3468] training loss: 1.922\n",
      "\n",
      "[iteration - 3478] training loss: 1.913\n",
      "\n",
      "[iteration - 3488] training loss: 1.924\n",
      "\n",
      "[iteration - 3498] training loss: 1.914\n",
      "\n",
      "[iteration - 3508] training loss: 1.917\n",
      "\n",
      "[iteration - 3518] training loss: 1.908\n",
      "\n",
      "[iteration - 3528] training loss: 1.901\n",
      "\n",
      "[iteration - 3538] training loss: 1.915\n",
      "\n",
      "[iteration - 3548] training loss: 1.893\n",
      "\n",
      "[iteration - 3558] training loss: 1.908\n",
      "\n",
      "[iteration - 3568] training loss: 1.883\n",
      "\n",
      "[iteration - 3578] training loss: 1.890\n",
      "\n",
      "[iteration - 3588] training loss: 1.908\n",
      "\n",
      "[iteration - 3598] training loss: 1.870\n",
      "\n",
      "[iteration - 3608] training loss: 1.915\n",
      "\n",
      "[iteration - 3618] training loss: 1.900\n",
      "\n",
      "[iteration - 3628] training loss: 1.895\n",
      "\n",
      "[iteration - 3638] training loss: 1.916\n",
      "\n",
      "[iteration - 3648] training loss: 1.919\n",
      "\n",
      "[iteration - 3658] training loss: 1.907\n",
      "\n",
      "[iteration - 3668] training loss: 1.885\n",
      "\n",
      "[iteration - 3678] training loss: 1.908\n",
      "\n",
      "[iteration - 3688] training loss: 1.920\n",
      "\n",
      "[iteration - 3698] training loss: 1.909\n",
      "\n",
      "[iteration - 3708] training loss: 1.887\n",
      "\n",
      "[iteration - 3718] training loss: 1.908\n",
      "\n",
      "[iteration - 3728] training loss: 1.905\n",
      "\n",
      "[iteration - 3738] training loss: 1.918\n",
      "\n",
      "[iteration - 3748] training loss: 1.920\n",
      "\n",
      "[iteration - 3758] training loss: 1.884\n",
      "\n",
      "[iteration - 3768] training loss: 1.900\n",
      "\n",
      "[iteration - 3778] training loss: 1.890\n",
      "\n",
      "[iteration - 3788] training loss: 1.899\n",
      "\n",
      "[iteration - 3798] training loss: 1.875\n",
      "\n",
      "[iteration - 3808] training loss: 1.906\n",
      "\n",
      "[iteration - 3818] training loss: 1.901\n",
      "\n",
      "[iteration - 3828] training loss: 1.884\n",
      "\n",
      "[iteration - 3838] training loss: 1.891\n",
      "\n",
      "[iteration - 3848] training loss: 1.908\n",
      "\n",
      "[iteration - 3858] training loss: 1.881\n",
      "\n",
      "[iteration - 3868] training loss: 1.866\n",
      "\n",
      "[iteration - 3878] training loss: 1.884\n",
      "\n",
      "[iteration - 3888] training loss: 1.899\n",
      "\n",
      "[iteration - 3898] training loss: 1.898\n",
      "\n",
      "[iteration - 3908] training loss: 1.896\n",
      "\n",
      "[iteration - 3910] training loss: 0.389\n",
      "\n",
      "[iteration - 3920] training loss: 1.870\n",
      "\n",
      "[iteration - 3930] training loss: 1.898\n",
      "\n",
      "[iteration - 3940] training loss: 1.887\n",
      "\n",
      "[iteration - 3950] training loss: 1.888\n",
      "\n",
      "[iteration - 3960] training loss: 1.884\n",
      "\n",
      "[iteration - 3970] training loss: 1.868\n",
      "\n",
      "[iteration - 3980] training loss: 1.865\n",
      "\n",
      "[iteration - 3990] training loss: 1.857\n",
      "\n",
      "[iteration - 4000] training loss: 1.864\n",
      "\n",
      "[iteration - 4010] training loss: 1.897\n",
      "\n",
      "[iteration - 4020] training loss: 1.889\n",
      "\n",
      "[iteration - 4030] training loss: 1.895\n",
      "\n",
      "[iteration - 4040] training loss: 1.884\n",
      "\n",
      "[iteration - 4050] training loss: 1.870\n",
      "\n",
      "[iteration - 4060] training loss: 1.894\n",
      "\n",
      "[iteration - 4070] training loss: 1.889\n",
      "\n",
      "[iteration - 4080] training loss: 1.879\n",
      "\n",
      "[iteration - 4090] training loss: 1.880\n",
      "\n",
      "[iteration - 4100] training loss: 1.897\n",
      "\n",
      "[iteration - 4110] training loss: 1.875\n",
      "\n",
      "[iteration - 4120] training loss: 1.879\n",
      "\n",
      "[iteration - 4130] training loss: 1.884\n",
      "\n",
      "[iteration - 4140] training loss: 1.885\n",
      "\n",
      "[iteration - 4150] training loss: 1.860\n",
      "\n",
      "[iteration - 4160] training loss: 1.888\n",
      "\n",
      "[iteration - 4170] training loss: 1.897\n",
      "\n",
      "[iteration - 4180] training loss: 1.908\n",
      "\n",
      "[iteration - 4190] training loss: 1.896\n",
      "\n",
      "[iteration - 4200] training loss: 1.864\n",
      "\n",
      "[iteration - 4210] training loss: 1.880\n",
      "\n",
      "[iteration - 4220] training loss: 1.848\n",
      "\n",
      "[iteration - 4230] training loss: 1.882\n",
      "\n",
      "[iteration - 4240] training loss: 1.866\n",
      "\n",
      "[iteration - 4250] training loss: 1.906\n",
      "\n",
      "[iteration - 4260] training loss: 1.865\n",
      "\n",
      "[iteration - 4270] training loss: 1.846\n",
      "\n",
      "[iteration - 4280] training loss: 1.891\n",
      "\n",
      "[iteration - 4290] training loss: 1.870\n",
      "\n",
      "[iteration - 4300] training loss: 1.888\n",
      "\n",
      "[iteration - 4310] training loss: 1.861\n",
      "\n",
      "[iteration - 4320] training loss: 1.853\n",
      "\n",
      "[iteration - 4330] training loss: 1.835\n",
      "\n",
      "[iteration - 4340] training loss: 1.864\n",
      "\n",
      "[iteration - 4350] training loss: 1.853\n",
      "\n",
      "[iteration - 4360] training loss: 1.873\n",
      "\n",
      "[iteration - 4370] training loss: 1.871\n",
      "\n",
      "[iteration - 4380] training loss: 1.892\n",
      "\n",
      "[iteration - 4390] training loss: 1.873\n",
      "\n",
      "[iteration - 4400] training loss: 1.885\n",
      "\n",
      "[iteration - 4410] training loss: 1.893\n",
      "\n",
      "[iteration - 4420] training loss: 1.897\n",
      "\n",
      "[iteration - 4430] training loss: 1.890\n",
      "\n",
      "[iteration - 4440] training loss: 1.861\n",
      "\n",
      "[iteration - 4450] training loss: 1.854\n",
      "\n",
      "[iteration - 4460] training loss: 1.831\n",
      "\n",
      "[iteration - 4470] training loss: 1.849\n",
      "\n",
      "[iteration - 4480] training loss: 1.882\n",
      "\n",
      "[iteration - 4490] training loss: 1.853\n",
      "\n",
      "[iteration - 4500] training loss: 1.859\n",
      "\n",
      "[iteration - 4510] training loss: 1.853\n",
      "\n",
      "[iteration - 4520] training loss: 1.870\n",
      "\n",
      "[iteration - 4530] training loss: 1.889\n",
      "\n",
      "[iteration - 4540] training loss: 1.876\n",
      "\n",
      "[iteration - 4550] training loss: 1.845\n",
      "\n",
      "[iteration - 4560] training loss: 1.895\n",
      "\n",
      "[iteration - 4570] training loss: 1.867\n",
      "\n",
      "[iteration - 4580] training loss: 1.858\n",
      "\n",
      "[iteration - 4590] training loss: 1.851\n",
      "\n",
      "[iteration - 4600] training loss: 1.880\n",
      "\n",
      "[iteration - 4610] training loss: 1.876\n",
      "\n",
      "[iteration - 4620] training loss: 1.854\n",
      "\n",
      "[iteration - 4630] training loss: 1.842\n",
      "\n",
      "[iteration - 4640] training loss: 1.851\n",
      "\n",
      "[iteration - 4650] training loss: 1.858\n",
      "\n",
      "[iteration - 4660] training loss: 1.877\n",
      "\n",
      "[iteration - 4670] training loss: 1.865\n",
      "\n",
      "[iteration - 4680] training loss: 1.850\n",
      "\n",
      "[iteration - 4690] training loss: 1.853\n",
      "\n",
      "[iteration - 4692] training loss: 0.381\n",
      "\n",
      "[iteration - 4702] training loss: 1.827\n",
      "\n",
      "[iteration - 4712] training loss: 1.852\n",
      "\n",
      "[iteration - 4722] training loss: 1.845\n",
      "\n",
      "[iteration - 4732] training loss: 1.849\n",
      "\n",
      "[iteration - 4742] training loss: 1.862\n",
      "\n",
      "[iteration - 4752] training loss: 1.845\n",
      "\n",
      "[iteration - 4762] training loss: 1.825\n",
      "\n",
      "[iteration - 4772] training loss: 1.866\n",
      "\n",
      "[iteration - 4782] training loss: 1.865\n",
      "\n",
      "[iteration - 4792] training loss: 1.864\n",
      "\n",
      "[iteration - 4802] training loss: 1.851\n",
      "\n",
      "[iteration - 4812] training loss: 1.865\n",
      "\n",
      "[iteration - 4822] training loss: 1.893\n",
      "\n",
      "[iteration - 4832] training loss: 1.861\n",
      "\n",
      "[iteration - 4842] training loss: 1.851\n",
      "\n",
      "[iteration - 4852] training loss: 1.841\n",
      "\n",
      "[iteration - 4862] training loss: 1.850\n",
      "\n",
      "[iteration - 4872] training loss: 1.844\n",
      "\n",
      "[iteration - 4882] training loss: 1.888\n",
      "\n",
      "[iteration - 4892] training loss: 1.846\n",
      "\n",
      "[iteration - 4902] training loss: 1.843\n",
      "\n",
      "[iteration - 4912] training loss: 1.867\n",
      "\n",
      "[iteration - 4922] training loss: 1.848\n",
      "\n",
      "[iteration - 4932] training loss: 1.857\n",
      "\n",
      "[iteration - 4942] training loss: 1.852\n",
      "\n",
      "[iteration - 4952] training loss: 1.851\n",
      "\n",
      "[iteration - 4962] training loss: 1.862\n",
      "\n",
      "[iteration - 4972] training loss: 1.846\n",
      "\n",
      "[iteration - 4982] training loss: 1.839\n",
      "\n",
      "[iteration - 4992] training loss: 1.829\n",
      "\n",
      "[iteration - 5002] training loss: 1.838\n",
      "\n",
      "[iteration - 5012] training loss: 1.882\n",
      "\n",
      "[iteration - 5022] training loss: 1.879\n",
      "\n",
      "[iteration - 5032] training loss: 1.826\n",
      "\n",
      "[iteration - 5042] training loss: 1.851\n",
      "\n",
      "[iteration - 5052] training loss: 1.847\n",
      "\n",
      "[iteration - 5062] training loss: 1.848\n",
      "\n",
      "[iteration - 5072] training loss: 1.855\n",
      "\n",
      "[iteration - 5082] training loss: 1.853\n",
      "\n",
      "[iteration - 5092] training loss: 1.824\n",
      "\n",
      "[iteration - 5102] training loss: 1.875\n",
      "\n",
      "[iteration - 5112] training loss: 1.851\n",
      "\n",
      "[iteration - 5122] training loss: 1.846\n",
      "\n",
      "[iteration - 5132] training loss: 1.851\n",
      "\n",
      "[iteration - 5142] training loss: 1.836\n",
      "\n",
      "[iteration - 5152] training loss: 1.846\n",
      "\n",
      "[iteration - 5162] training loss: 1.849\n",
      "\n",
      "[iteration - 5172] training loss: 1.838\n",
      "\n",
      "[iteration - 5182] training loss: 1.834\n",
      "\n",
      "[iteration - 5192] training loss: 1.841\n",
      "\n",
      "[iteration - 5202] training loss: 1.854\n",
      "\n",
      "[iteration - 5212] training loss: 1.864\n",
      "\n",
      "[iteration - 5222] training loss: 1.832\n",
      "\n",
      "[iteration - 5232] training loss: 1.822\n",
      "\n",
      "[iteration - 5242] training loss: 1.854\n",
      "\n",
      "[iteration - 5252] training loss: 1.823\n",
      "\n",
      "[iteration - 5262] training loss: 1.812\n",
      "\n",
      "[iteration - 5272] training loss: 1.848\n",
      "\n",
      "[iteration - 5282] training loss: 1.838\n",
      "\n",
      "[iteration - 5292] training loss: 1.845\n",
      "\n",
      "[iteration - 5302] training loss: 1.831\n",
      "\n",
      "[iteration - 5312] training loss: 1.872\n",
      "\n",
      "[iteration - 5322] training loss: 1.847\n",
      "\n",
      "[iteration - 5332] training loss: 1.894\n",
      "\n",
      "[iteration - 5342] training loss: 1.865\n",
      "\n",
      "[iteration - 5352] training loss: 1.869\n",
      "\n",
      "[iteration - 5362] training loss: 1.861\n",
      "\n",
      "[iteration - 5372] training loss: 1.848\n",
      "\n",
      "[iteration - 5382] training loss: 1.832\n",
      "\n",
      "[iteration - 5392] training loss: 1.848\n",
      "\n",
      "[iteration - 5402] training loss: 1.872\n",
      "\n",
      "[iteration - 5412] training loss: 1.833\n",
      "\n",
      "[iteration - 5422] training loss: 1.860\n",
      "\n",
      "[iteration - 5432] training loss: 1.864\n",
      "\n",
      "[iteration - 5442] training loss: 1.868\n",
      "\n",
      "[iteration - 5452] training loss: 1.854\n",
      "\n",
      "[iteration - 5462] training loss: 1.855\n",
      "\n",
      "[iteration - 5472] training loss: 1.847\n",
      "\n",
      "[iteration - 5474] training loss: 0.362\n",
      "\n",
      "[iteration - 5484] training loss: 1.831\n",
      "\n",
      "[iteration - 5494] training loss: 1.819\n",
      "\n",
      "[iteration - 5504] training loss: 1.832\n",
      "\n",
      "[iteration - 5514] training loss: 1.874\n",
      "\n",
      "[iteration - 5524] training loss: 1.834\n",
      "\n",
      "[iteration - 5534] training loss: 1.842\n",
      "\n",
      "[iteration - 5544] training loss: 1.832\n",
      "\n",
      "[iteration - 5554] training loss: 1.837\n",
      "\n",
      "[iteration - 5564] training loss: 1.820\n",
      "\n",
      "[iteration - 5574] training loss: 1.840\n",
      "\n",
      "[iteration - 5584] training loss: 1.837\n",
      "\n",
      "[iteration - 5594] training loss: 1.812\n",
      "\n",
      "[iteration - 5604] training loss: 1.855\n",
      "\n",
      "[iteration - 5614] training loss: 1.839\n",
      "\n",
      "[iteration - 5624] training loss: 1.823\n",
      "\n",
      "[iteration - 5634] training loss: 1.815\n",
      "\n",
      "[iteration - 5644] training loss: 1.828\n",
      "\n",
      "[iteration - 5654] training loss: 1.823\n",
      "\n",
      "[iteration - 5664] training loss: 1.859\n",
      "\n",
      "[iteration - 5674] training loss: 1.823\n",
      "\n",
      "[iteration - 5684] training loss: 1.835\n",
      "\n",
      "[iteration - 5694] training loss: 1.859\n",
      "\n",
      "[iteration - 5704] training loss: 1.831\n",
      "\n",
      "[iteration - 5714] training loss: 1.827\n",
      "\n",
      "[iteration - 5724] training loss: 1.838\n",
      "\n",
      "[iteration - 5734] training loss: 1.873\n",
      "\n",
      "[iteration - 5744] training loss: 1.829\n",
      "\n",
      "[iteration - 5754] training loss: 1.820\n",
      "\n",
      "[iteration - 5764] training loss: 1.846\n",
      "\n",
      "[iteration - 5774] training loss: 1.821\n",
      "\n",
      "[iteration - 5784] training loss: 1.828\n",
      "\n",
      "[iteration - 5794] training loss: 1.800\n",
      "\n",
      "[iteration - 5804] training loss: 1.816\n",
      "\n",
      "[iteration - 5814] training loss: 1.828\n",
      "\n",
      "[iteration - 5824] training loss: 1.822\n",
      "\n",
      "[iteration - 5834] training loss: 1.871\n",
      "\n",
      "[iteration - 5844] training loss: 1.803\n",
      "\n",
      "[iteration - 5854] training loss: 1.842\n",
      "\n",
      "[iteration - 5864] training loss: 1.830\n",
      "\n",
      "[iteration - 5874] training loss: 1.847\n",
      "\n",
      "[iteration - 5884] training loss: 1.849\n",
      "\n",
      "[iteration - 5894] training loss: 1.864\n",
      "\n",
      "[iteration - 5904] training loss: 1.839\n",
      "\n",
      "[iteration - 5914] training loss: 1.824\n",
      "\n",
      "[iteration - 5924] training loss: 1.822\n",
      "\n",
      "[iteration - 5934] training loss: 1.842\n",
      "\n",
      "[iteration - 5944] training loss: 1.833\n",
      "\n",
      "[iteration - 5954] training loss: 1.820\n",
      "\n",
      "[iteration - 5964] training loss: 1.847\n",
      "\n",
      "[iteration - 5974] training loss: 1.834\n",
      "\n",
      "[iteration - 5984] training loss: 1.824\n",
      "\n",
      "[iteration - 5994] training loss: 1.827\n",
      "\n",
      "[iteration - 6004] training loss: 1.805\n",
      "\n",
      "[iteration - 6014] training loss: 1.824\n",
      "\n",
      "[iteration - 6024] training loss: 1.816\n",
      "\n",
      "[iteration - 6034] training loss: 1.837\n",
      "\n",
      "[iteration - 6044] training loss: 1.835\n",
      "\n",
      "[iteration - 6054] training loss: 1.841\n",
      "\n",
      "[iteration - 6064] training loss: 1.804\n",
      "\n",
      "[iteration - 6074] training loss: 1.796\n",
      "\n",
      "[iteration - 6084] training loss: 1.832\n",
      "\n",
      "[iteration - 6094] training loss: 1.829\n",
      "\n",
      "[iteration - 6104] training loss: 1.842\n",
      "\n",
      "[iteration - 6114] training loss: 1.826\n",
      "\n",
      "[iteration - 6124] training loss: 1.838\n",
      "\n",
      "[iteration - 6134] training loss: 1.832\n",
      "\n",
      "[iteration - 6144] training loss: 1.816\n",
      "\n",
      "[iteration - 6154] training loss: 1.838\n",
      "\n",
      "[iteration - 6164] training loss: 1.849\n",
      "\n",
      "[iteration - 6174] training loss: 1.815\n",
      "\n",
      "[iteration - 6184] training loss: 1.831\n",
      "\n",
      "[iteration - 6194] training loss: 1.782\n",
      "\n",
      "[iteration - 6204] training loss: 1.823\n",
      "\n",
      "[iteration - 6214] training loss: 1.835\n",
      "\n",
      "[iteration - 6224] training loss: 1.834\n",
      "\n",
      "[iteration - 6234] training loss: 1.823\n",
      "\n",
      "[iteration - 6244] training loss: 1.849\n",
      "\n",
      "[iteration - 6254] training loss: 1.804\n",
      "\n",
      "[iteration - 6256] training loss: 0.364\n",
      "\n",
      "[iteration - 6266] training loss: 1.832\n",
      "\n",
      "[iteration - 6276] training loss: 1.825\n",
      "\n",
      "[iteration - 6286] training loss: 1.811\n",
      "\n",
      "[iteration - 6296] training loss: 1.801\n",
      "\n",
      "[iteration - 6306] training loss: 1.827\n",
      "\n",
      "[iteration - 6316] training loss: 1.822\n",
      "\n",
      "[iteration - 6326] training loss: 1.827\n",
      "\n",
      "[iteration - 6336] training loss: 1.817\n",
      "\n",
      "[iteration - 6346] training loss: 1.812\n",
      "\n",
      "[iteration - 6356] training loss: 1.838\n",
      "\n",
      "[iteration - 6366] training loss: 1.799\n",
      "\n",
      "[iteration - 6376] training loss: 1.778\n",
      "\n",
      "[iteration - 6386] training loss: 1.855\n",
      "\n",
      "[iteration - 6396] training loss: 1.832\n",
      "\n",
      "[iteration - 6406] training loss: 1.837\n",
      "\n",
      "[iteration - 6416] training loss: 1.832\n",
      "\n",
      "[iteration - 6426] training loss: 1.840\n",
      "\n",
      "[iteration - 6436] training loss: 1.766\n",
      "\n",
      "[iteration - 6446] training loss: 1.807\n",
      "\n",
      "[iteration - 6456] training loss: 1.796\n",
      "\n",
      "[iteration - 6466] training loss: 1.801\n",
      "\n",
      "[iteration - 6476] training loss: 1.827\n",
      "\n",
      "[iteration - 6486] training loss: 1.809\n",
      "\n",
      "[iteration - 6496] training loss: 1.806\n",
      "\n",
      "[iteration - 6506] training loss: 1.801\n",
      "\n",
      "[iteration - 6516] training loss: 1.799\n",
      "\n",
      "[iteration - 6526] training loss: 1.836\n",
      "\n",
      "[iteration - 6536] training loss: 1.815\n",
      "\n",
      "[iteration - 6546] training loss: 1.825\n",
      "\n",
      "[iteration - 6556] training loss: 1.825\n",
      "\n",
      "[iteration - 6566] training loss: 1.792\n",
      "\n",
      "[iteration - 6576] training loss: 1.811\n",
      "\n",
      "[iteration - 6586] training loss: 1.824\n",
      "\n",
      "[iteration - 6596] training loss: 1.786\n",
      "\n",
      "[iteration - 6606] training loss: 1.823\n",
      "\n",
      "[iteration - 6616] training loss: 1.809\n",
      "\n",
      "[iteration - 6626] training loss: 1.840\n",
      "\n",
      "[iteration - 6636] training loss: 1.830\n",
      "\n",
      "[iteration - 6646] training loss: 1.822\n",
      "\n",
      "[iteration - 6656] training loss: 1.851\n",
      "\n",
      "[iteration - 6666] training loss: 1.791\n",
      "\n",
      "[iteration - 6676] training loss: 1.798\n",
      "\n",
      "[iteration - 6686] training loss: 1.794\n",
      "\n",
      "[iteration - 6696] training loss: 1.821\n",
      "\n",
      "[iteration - 6706] training loss: 1.835\n",
      "\n",
      "[iteration - 6716] training loss: 1.800\n",
      "\n",
      "[iteration - 6726] training loss: 1.837\n",
      "\n",
      "[iteration - 6736] training loss: 1.823\n",
      "\n",
      "[iteration - 6746] training loss: 1.813\n",
      "\n",
      "[iteration - 6756] training loss: 1.798\n",
      "\n",
      "[iteration - 6766] training loss: 1.857\n",
      "\n",
      "[iteration - 6776] training loss: 1.844\n",
      "\n",
      "[iteration - 6786] training loss: 1.809\n",
      "\n",
      "[iteration - 6796] training loss: 1.835\n",
      "\n",
      "[iteration - 6806] training loss: 1.815\n",
      "\n",
      "[iteration - 6816] training loss: 1.807\n",
      "\n",
      "[iteration - 6826] training loss: 1.828\n",
      "\n",
      "[iteration - 6836] training loss: 1.814\n",
      "\n",
      "[iteration - 6846] training loss: 1.808\n",
      "\n",
      "[iteration - 6856] training loss: 1.849\n",
      "\n",
      "[iteration - 6866] training loss: 1.834\n",
      "\n",
      "[iteration - 6876] training loss: 1.815\n",
      "\n",
      "[iteration - 6886] training loss: 1.784\n",
      "\n",
      "[iteration - 6896] training loss: 1.832\n",
      "\n",
      "[iteration - 6906] training loss: 1.833\n",
      "\n",
      "[iteration - 6916] training loss: 1.808\n",
      "\n",
      "[iteration - 6926] training loss: 1.817\n",
      "\n",
      "[iteration - 6936] training loss: 1.835\n",
      "\n",
      "[iteration - 6946] training loss: 1.797\n",
      "\n",
      "[iteration - 6956] training loss: 1.823\n",
      "\n",
      "[iteration - 6966] training loss: 1.815\n",
      "\n",
      "[iteration - 6976] training loss: 1.811\n",
      "\n",
      "[iteration - 6986] training loss: 1.821\n",
      "\n",
      "[iteration - 6996] training loss: 1.805\n",
      "\n",
      "[iteration - 7006] training loss: 1.797\n",
      "\n",
      "[iteration - 7016] training loss: 1.810\n",
      "\n",
      "[iteration - 7026] training loss: 1.799\n",
      "\n",
      "[iteration - 7036] training loss: 1.794\n",
      "\n",
      "[iteration - 7038] training loss: 0.382\n",
      "\n",
      "[iteration - 7048] training loss: 1.846\n",
      "\n",
      "[iteration - 7058] training loss: 1.816\n",
      "\n",
      "[iteration - 7068] training loss: 1.783\n",
      "\n",
      "[iteration - 7078] training loss: 1.819\n",
      "\n",
      "[iteration - 7088] training loss: 1.814\n",
      "\n",
      "[iteration - 7098] training loss: 1.798\n",
      "\n",
      "[iteration - 7108] training loss: 1.796\n",
      "\n",
      "[iteration - 7118] training loss: 1.797\n",
      "\n",
      "[iteration - 7128] training loss: 1.808\n",
      "\n",
      "[iteration - 7138] training loss: 1.820\n",
      "\n",
      "[iteration - 7148] training loss: 1.830\n",
      "\n",
      "[iteration - 7158] training loss: 1.800\n",
      "\n",
      "[iteration - 7168] training loss: 1.799\n",
      "\n",
      "[iteration - 7178] training loss: 1.807\n",
      "\n",
      "[iteration - 7188] training loss: 1.817\n",
      "\n",
      "[iteration - 7198] training loss: 1.783\n",
      "\n",
      "[iteration - 7208] training loss: 1.793\n",
      "\n",
      "[iteration - 7218] training loss: 1.819\n",
      "\n",
      "[iteration - 7228] training loss: 1.782\n",
      "\n",
      "[iteration - 7238] training loss: 1.771\n",
      "\n",
      "[iteration - 7248] training loss: 1.820\n",
      "\n",
      "[iteration - 7258] training loss: 1.816\n",
      "\n",
      "[iteration - 7268] training loss: 1.790\n",
      "\n",
      "[iteration - 7278] training loss: 1.794\n",
      "\n",
      "[iteration - 7288] training loss: 1.826\n",
      "\n",
      "[iteration - 7298] training loss: 1.818\n",
      "\n",
      "[iteration - 7308] training loss: 1.785\n",
      "\n",
      "[iteration - 7318] training loss: 1.788\n",
      "\n",
      "[iteration - 7328] training loss: 1.810\n",
      "\n",
      "[iteration - 7338] training loss: 1.811\n",
      "\n",
      "[iteration - 7348] training loss: 1.794\n",
      "\n",
      "[iteration - 7358] training loss: 1.781\n",
      "\n",
      "[iteration - 7368] training loss: 1.773\n",
      "\n",
      "[iteration - 7378] training loss: 1.798\n",
      "\n",
      "[iteration - 7388] training loss: 1.804\n",
      "\n",
      "[iteration - 7398] training loss: 1.830\n",
      "\n",
      "[iteration - 7408] training loss: 1.816\n",
      "\n",
      "[iteration - 7418] training loss: 1.810\n",
      "\n",
      "[iteration - 7428] training loss: 1.842\n",
      "\n",
      "[iteration - 7438] training loss: 1.774\n",
      "\n",
      "[iteration - 7448] training loss: 1.810\n",
      "\n",
      "[iteration - 7458] training loss: 1.799\n",
      "\n",
      "[iteration - 7468] training loss: 1.804\n",
      "\n",
      "[iteration - 7478] training loss: 1.817\n",
      "\n",
      "[iteration - 7488] training loss: 1.819\n",
      "\n",
      "[iteration - 7498] training loss: 1.815\n",
      "\n",
      "[iteration - 7508] training loss: 1.802\n",
      "\n",
      "[iteration - 7518] training loss: 1.798\n",
      "\n",
      "[iteration - 7528] training loss: 1.796\n",
      "\n",
      "[iteration - 7538] training loss: 1.800\n",
      "\n",
      "[iteration - 7548] training loss: 1.800\n",
      "\n",
      "[iteration - 7558] training loss: 1.828\n",
      "\n",
      "[iteration - 7568] training loss: 1.801\n",
      "\n",
      "[iteration - 7578] training loss: 1.801\n",
      "\n",
      "[iteration - 7588] training loss: 1.794\n",
      "\n",
      "[iteration - 7598] training loss: 1.790\n",
      "\n",
      "[iteration - 7608] training loss: 1.823\n",
      "\n",
      "[iteration - 7618] training loss: 1.823\n",
      "\n",
      "[iteration - 7628] training loss: 1.798\n",
      "\n",
      "[iteration - 7638] training loss: 1.785\n",
      "\n",
      "[iteration - 7648] training loss: 1.798\n",
      "\n",
      "[iteration - 7658] training loss: 1.796\n",
      "\n",
      "[iteration - 7668] training loss: 1.792\n",
      "\n",
      "[iteration - 7678] training loss: 1.793\n",
      "\n",
      "[iteration - 7688] training loss: 1.804\n",
      "\n",
      "[iteration - 7698] training loss: 1.804\n",
      "\n",
      "[iteration - 7708] training loss: 1.781\n",
      "\n",
      "[iteration - 7718] training loss: 1.808\n",
      "\n",
      "[iteration - 7728] training loss: 1.827\n",
      "\n",
      "[iteration - 7738] training loss: 1.791\n",
      "\n",
      "[iteration - 7748] training loss: 1.806\n",
      "\n",
      "[iteration - 7758] training loss: 1.793\n",
      "\n",
      "[iteration - 7768] training loss: 1.817\n",
      "\n",
      "[iteration - 7778] training loss: 1.793\n",
      "\n",
      "[iteration - 7788] training loss: 1.790\n",
      "\n",
      "[iteration - 7798] training loss: 1.812\n",
      "\n",
      "[iteration - 7808] training loss: 1.821\n",
      "\n",
      "[iteration - 7818] training loss: 1.800\n",
      "\n",
      "Accuracy of the network on the 10000 test images: 65.22% \n",
      "\n"
     ]
    }
   ],
   "source": [
    "model=create_model(0,32).to(device)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.9)\n",
    "loss_f = nn.CrossEntropyLoss()\n",
    "Epochs=10\n",
    "normal_train(model, optimizer, loss_f, Epochs)\n",
    "test_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the network on the 10000 test images: 65.16% \n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=================================================================\n",
      "Layer (type:depth-idx)                   Param #\n",
      "=================================================================\n",
      "├─Conv2d: 1-1                            4,704\n",
      "├─BatchNorm2d: 1-2                       64\n",
      "├─ReLU: 1-3                              --\n",
      "├─MaxPool2d: 1-4                         --\n",
      "├─Conv2d: 1-5                            9,216\n",
      "├─BatchNorm2d: 1-6                       64\n",
      "├─ReLU: 1-7                              --\n",
      "├─MaxPool2d: 1-8                         --\n",
      "├─Conv2d: 1-9                            9,216\n",
      "├─BatchNorm2d: 1-10                      64\n",
      "├─ReLU: 1-11                             --\n",
      "├─MaxPool2d: 1-12                        --\n",
      "├─Conv2d: 1-13                           9,216\n",
      "├─BatchNorm2d: 1-14                      64\n",
      "├─ReLU: 1-15                             --\n",
      "├─MaxPool2d: 1-16                        --\n",
      "├─Conv2d: 1-17                           9,216\n",
      "├─BatchNorm2d: 1-18                      64\n",
      "├─ReLU: 1-19                             --\n",
      "├─MaxPool2d: 1-20                        --\n",
      "├─Conv2d: 1-21                           9,216\n",
      "├─BatchNorm2d: 1-22                      64\n",
      "├─ReLU: 1-23                             --\n",
      "├─MaxPool2d: 1-24                        --\n",
      "├─Conv2d: 1-25                           9,216\n",
      "├─BatchNorm2d: 1-26                      64\n",
      "├─ReLU: 1-27                             --\n",
      "├─MaxPool2d: 1-28                        --\n",
      "├─Flatten: 1-29                          --\n",
      "├─Linear: 1-30                           1,290\n",
      "├─Softmax: 1-31                          --\n",
      "=================================================================\n",
      "Total params: 61,738\n",
      "Trainable params: 61,738\n",
      "Non-trainable params: 0\n",
      "=================================================================\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "=================================================================\n",
       "Layer (type:depth-idx)                   Param #\n",
       "=================================================================\n",
       "├─Conv2d: 1-1                            4,704\n",
       "├─BatchNorm2d: 1-2                       64\n",
       "├─ReLU: 1-3                              --\n",
       "├─MaxPool2d: 1-4                         --\n",
       "├─Conv2d: 1-5                            9,216\n",
       "├─BatchNorm2d: 1-6                       64\n",
       "├─ReLU: 1-7                              --\n",
       "├─MaxPool2d: 1-8                         --\n",
       "├─Conv2d: 1-9                            9,216\n",
       "├─BatchNorm2d: 1-10                      64\n",
       "├─ReLU: 1-11                             --\n",
       "├─MaxPool2d: 1-12                        --\n",
       "├─Conv2d: 1-13                           9,216\n",
       "├─BatchNorm2d: 1-14                      64\n",
       "├─ReLU: 1-15                             --\n",
       "├─MaxPool2d: 1-16                        --\n",
       "├─Conv2d: 1-17                           9,216\n",
       "├─BatchNorm2d: 1-18                      64\n",
       "├─ReLU: 1-19                             --\n",
       "├─MaxPool2d: 1-20                        --\n",
       "├─Conv2d: 1-21                           9,216\n",
       "├─BatchNorm2d: 1-22                      64\n",
       "├─ReLU: 1-23                             --\n",
       "├─MaxPool2d: 1-24                        --\n",
       "├─Conv2d: 1-25                           9,216\n",
       "├─BatchNorm2d: 1-26                      64\n",
       "├─ReLU: 1-27                             --\n",
       "├─MaxPool2d: 1-28                        --\n",
       "├─Flatten: 1-29                          --\n",
       "├─Linear: 1-30                           1,290\n",
       "├─Softmax: 1-31                          --\n",
       "=================================================================\n",
       "Total params: 61,738\n",
       "Trainable params: 61,738\n",
       "Non-trainable params: 0\n",
       "================================================================="
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary(model, input_size=(3,224,224))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}