seed_v.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: cuda:0\n"
     ]
    }
   ],
   "source": [
    "## Standard libraries\n",
    "import os\n",
    "import numpy as np\n",
    "import random\n",
    "import json\n",
    "from PIL import Image\n",
    "from collections import defaultdict\n",
    "from statistics import mean, stdev\n",
    "from copy import deepcopy\n",
    "import pickle\n",
    "import re\n",
    "\n",
    "## Imports for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "plt.set_cmap('cividis')\n",
    "%matplotlib inline\n",
    "from IPython.display import set_matplotlib_formats\n",
    "import matplotlib\n",
    "matplotlib.rcParams['lines.linewidth'] = 2.0\n",
    "import seaborn as sns\n",
    "sns.reset_orig()\n",
    "\n",
    "## tqdm for loading bars\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "## PyTorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.utils.data as data\n",
    "import torch.optim as optim\n",
    "\n",
    "## Torchvision\n",
    "import torchvision\n",
    "from torchvision.datasets import CIFAR100, SVHN\n",
    "from torchvision import transforms\n",
    "\n",
    "# PyTorch Lightning\n",
    "import pytorch_lightning as pl\n",
    "from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint\n",
    "pl.seed_everything(42)\n",
    "\n",
    "# Import tensorboard\n",
    "%load_ext tensorboard\n",
    "\n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(\"Device:\", device)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set folder paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path to the folder where the datasets are/should be downloaded (e.g. CIFAR10)\n",
    "DATASET_PATH = \"raw_data/seed-v/merged_data\"\n",
    "# Path to the folder where the pretrained models are saved\n",
    "CHECKPOINT_PATH = \"saved_models/seed-v\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define dataset class for SEEDV and initialize a new dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SEEDV(data.Dataset):\n",
    "    def __init__(self, emotion_dict, num_participants, data_dir):\n",
    "        self.emotion_dict = emotion_dict\n",
    "        self.num_participants = num_participants\n",
    "        self.data_dir = data_dir\n",
    "        self.tensor_dataset = self.load_data()\n",
    "    \n",
    "    def load_data(self):\n",
    "        dataset = None           \n",
    "        for file in os.listdir(self.data_dir):\n",
    "            file_path = os.path.join(self.data_dir, file)\n",
    "            if dataset is None:\n",
    "                target = np.int64(re.findall(\"\\d+\", file)[0])\n",
    "                for i in range(1, len(np.load(file_path))):\n",
    "                    target = np.hstack((target, int(re.findall(\"\\d+\", file)[0])))\n",
    "                dataset = np.load(file_path)\n",
    "            else:\n",
    "                for i in range(len(np.load(file_path))):\n",
    "                    target = np.hstack((target, np.int64(re.findall(\"\\d+\", file)[0])))\n",
    "                dataset = np.vstack((dataset, np.load(file_path)))\n",
    "\n",
    "        tensor_dataset = data.TensorDataset(torch.from_numpy(dataset[:, :-1]), torch.from_numpy(dataset[:, -1]), torch.from_numpy(target))\n",
    "                    \n",
    "        return tensor_dataset\n",
    "    \n",
    "    def get_all_data(self):\n",
    "        \"\"\"\n",
    "        same as self.__getitem__(self, idx) but instead of a specific index\n",
    "        this function will return all the tuple that contains all features and \n",
    "        combined labels \n",
    "        \"\"\"\n",
    "        all_features = None\n",
    "        all_combined_targets = None\n",
    "\n",
    "        for idx in range(len(self.tensor_dataset)):\n",
    "            if all_features == None:\n",
    "                all_features, all_combined_targets = self.__getitem__(idx)\n",
    "            else:\n",
    "                features, combined_targets = self.__getitem__(idx)\n",
    "                all_features = torch.vstack((all_features, features))\n",
    "                all_combined_targets = torch.vstack((all_combined_targets, combined_targets))\n",
    "        \n",
    "        return all_features, all_combined_targets\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.tensor_dataset)\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        \"\"\"\n",
    "        return a tuple of features, combined_label (from participant and emotion)\n",
    "        p1: 0 1 2 3 4,\n",
    "        p2: 5 6 7 8 9,\n",
    "        ...\n",
    "        p16: 75 76 77 78 79\n",
    "\n",
    "        we are only given participant # and emotion #\n",
    "        p1_0: 0 = (1 - 1) * 5\n",
    "        p1_1: 1 = 0 + 1\n",
    "        p1_2: 2 = 0 + 2\n",
    "        p1_3: 3 = 0 + 3\n",
    "        p1_4: 4 = 0 + 4\n",
    "        p2_0: 5 = (2 - 1) * 5\n",
    "        p2_1: 6 = 5 + 1\n",
    "        p2_2: 7 = 5 + 2\n",
    "        p2_3: 8 = 5 + 3\n",
    "        p2_4: 9 = 5 + 4\n",
    "        ..\n",
    "        p16_0: 75 = (16 - 1) * 5 = 75\n",
    "        ...\n",
    "        \"\"\"\n",
    "        features = self.tensor_dataset[idx][0]\n",
    "        emotion_num = self.tensor_dataset[idx][1]\n",
    "        participant_num = self.tensor_dataset[idx][2]\n",
    "        base = (participant_num - 1) * len(self.emotion_dict)\n",
    "        combined_target = base + emotion_num\n",
    "\n",
    "        return features, combined_target.to(torch.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_v = SEEDV(emotion_dict = {0: 'disgust', 1: 'fear', 2: 'sad', 3: 'neutral', 4: 'happy'}, num_participants=16, data_dir=DATASET_PATH)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform a train-val-test split by label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = torch.randperm(5*16) # Generate random permutation of numbers from 0 to 79\n",
    "train_classes, val_classes, test_classes = classes[:64], classes[64:72], classes[72:] # 80-10-10 split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEEDV_all_features, SEEDV_all_targets = seed_v.get_all_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EEGDataset(data.Dataset):\n",
    "    def __init__(self, features, targets):\n",
    "        super().__init__()\n",
    "        self.features = features\n",
    "        self.targets = targets\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        features, targets = self.features[idx], self.targets[idx]\n",
    "\n",
    "        return features, targets\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.features.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataset_from_labels(features, targets, class_set): \n",
    "    # for label in labels:\n",
    "    #     print(label)\n",
    "    class_mask = (targets[:,None] == class_set[None,:]).any(dim=-1) # reshape class mask [[64], [64],... ] -> [64, 64, ...]\n",
    "    return EEGDataset(features[class_mask], targets[class_mask]) # reshape labels [[0], [1], ...] -> [0, 1, ...]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = dataset_from_labels(\n",
    "    SEEDV_all_features, SEEDV_all_targets.reshape((1,-1))[0], train_classes)\n",
    "val_set = dataset_from_labels(\n",
    "    SEEDV_all_features, SEEDV_all_targets.reshape((1,-1))[0], val_classes)\n",
    "test_set = dataset_from_labels(\n",
    "    SEEDV_all_features, SEEDV_all_targets.reshape((1,-1))[0], test_classes)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup dataloaders and samplers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FewShotBatchSampler(object):\n",
    "\n",
    "    def __init__(self, dataset_targets, N_way, K_shot, include_query=False, shuffle=True, shuffle_once=False):\n",
    "        \"\"\"\n",
    "        Inputs:\n",
    "            dataset_targets - PyTorch tensor of the labels of the data elements.\n",
    "            N_way - Number of classes to sample per batch.\n",
    "            K_shot - Number of examples to sample per class in the batch.\n",
    "            include_query - If True, returns batch of size N_way*K_shot*2, which \n",
    "                            can be split into support and query set. Simplifies\n",
    "                            the implementation of sampling the same classes but \n",
    "                            distinct examples for support and query set.\n",
    "            shuffle - If True, examples and classes are newly shuffled in each\n",
    "                      iteration (for training)\n",
    "            shuffle_once - If True, examples and classes are shuffled once in \n",
    "                           the beginning, but kept constant across iterations \n",
    "                           (for validation)\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.dataset_targets = dataset_targets\n",
    "        self.N_way = N_way\n",
    "        self.K_shot = K_shot\n",
    "        self.shuffle = shuffle\n",
    "        self.include_query = include_query\n",
    "        if self.include_query:\n",
    "            self.K_shot *= 2\n",
    "        self.batch_size = self.N_way * self.K_shot  # Number of overall images per batch\n",
    "\n",
    "        # Organize examples by class\n",
    "        self.classes = torch.unique(self.dataset_targets).tolist()\n",
    "        self.num_classes = len(self.classes)\n",
    "        self.indices_per_class = {}\n",
    "        self.batches_per_class = {}  # Number of K-shot batches that each class can provide\n",
    "        for c in self.classes:\n",
    "            self.indices_per_class[c] = torch.where(self.dataset_targets == c)[0]\n",
    "            self.batches_per_class[c] = self.indices_per_class[c].shape[0] // self.K_shot\n",
    "\n",
    "        # Create a list of classes from which we select the N classes per batch\n",
    "        self.iterations = sum(self.batches_per_class.values()) // self.N_way\n",
    "        self.class_list = [c for c in self.classes for _ in range(self.batches_per_class[c])]\n",
    "        if shuffle_once or self.shuffle:\n",
    "            self.shuffle_data()\n",
    "        else:\n",
    "            # For testing, we iterate over classes instead of shuffling them\n",
    "            sort_idxs = [i+p*self.num_classes for i,\n",
    "                         c in enumerate(self.classes) for p in range(self.batches_per_class[c])]\n",
    "            self.class_list = np.array(self.class_list)[np.argsort(sort_idxs)].tolist()\n",
    "\n",
    "    def shuffle_data(self):\n",
    "        # Shuffle the examples per class\n",
    "        for c in self.classes:\n",
    "            perm = torch.randperm(self.indices_per_class[c].shape[0])\n",
    "            self.indices_per_class[c] = self.indices_per_class[c][perm]\n",
    "        # Shuffle the class list from which we sample. Note that this way of shuffling\n",
    "        # does not prevent to choose the same class twice in a batch. However, for \n",
    "        # training and validation, this is not a problem.\n",
    "        random.shuffle(self.class_list)\n",
    "\n",
    "    def __iter__(self):\n",
    "        # Shuffle data\n",
    "        if self.shuffle:\n",
    "            self.shuffle_data()\n",
    "\n",
    "        # Sample few-shot batches\n",
    "        start_index = defaultdict(int)\n",
    "        for it in range(self.iterations):\n",
    "            # Select N classes for the batch\n",
    "            class_batch = self.class_list[it*self.N_way:(it+1)*self.N_way]\n",
    "            index_batch = []\n",
    "            for c in class_batch:  # For each class, select the next K examples and add them to the batch\n",
    "                index_batch.extend(self.indices_per_class[c][start_index[c]:start_index[c]+self.K_shot])\n",
    "                start_index[c] += self.K_shot\n",
    "                try:\n",
    "                    self.indices_per_class[c][start_index[c]]\n",
    "                except:\n",
    "                    start_index[c] = 0\n",
    "            if self.include_query:  # If we return support+query set, sort them so that they are easy to split\n",
    "                index_batch = index_batch[::2] + index_batch[1::2]\n",
    "            yield index_batch\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.iterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_WAY = 5\n",
    "K_SHOT = 4\n",
    "train_data_loader = data.DataLoader(train_set,\n",
    "                                    batch_sampler=FewShotBatchSampler(train_set.targets,\n",
    "                                                                      include_query=True,\n",
    "                                                                      N_way=N_WAY,\n",
    "                                                                      K_shot=K_SHOT,\n",
    "                                                                      shuffle=True),\n",
    "                                    num_workers=32)\n",
    "val_data_loader = data.DataLoader(val_set,\n",
    "                                  batch_sampler=FewShotBatchSampler(val_set.targets,\n",
    "                                                                    include_query=True,\n",
    "                                                                    N_way=N_WAY,\n",
    "                                                                    K_shot=K_SHOT,\n",
    "                                                                    shuffle=False,\n",
    "                                                                    shuffle_once=True),\n",
    "                                  num_workers=32)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split batch into query and support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_query_support(features, targets):\n",
    "    support_features, query_features = features.chunk(2, dim=0)\n",
    "    support_targets, query_targets = targets.chunk(2, dim=0)\n",
    "    return support_features, query_features, support_targets, query_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "features, targets = next(iter(val_data_loader))\n",
    "support_features, query_features, support_targets, query_targets = split_query_support(features, targets)\n",
    "\n",
    "# fig, ax = plt.subplots(1, 2, figsize=(8, 5))\n",
    "# ax[0].plot(support_features, 'o')\n",
    "# ax[0].set_title(\"Support set\")\n",
    "# ax[0].axis('off')\n",
    "# ax[1].plot(query_features, 'o')\n",
    "# ax[1].set_title(\"Query set\")\n",
    "# ax[1].axis('off')\n",
    "# plt.suptitle(\"Few Shot Batch\", weight='bold')\n",
    "# plt.show()\n",
    "# plt.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize and visualize the features in batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize the features\n",
    "support_features = support_features / support_features.max(0, keepdim=True)[0]\n",
    "query_features = query_features / query_features.max(0, keepdim=True)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAn8AAAHHCAYAAADOPz5+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6RklEQVR4nO3deXQUZdr+8au6O+l0EiAhbGELgigMRkA22QyCDIogKuA6AiozijhuvC9uKOjrDK4jiII/nRkQR1xmRkFQBNlRAUEBWRRkBhCQPUDInnQ/vz9CWpqsSNKdTn0/5+ScpOruqrsbrVypp54qyxhjBAAAAFtwhLoBAAAABA/hDwAAwEYIfwAAADZC+AMAALARwh8AAICNEP4AAABshPAHAABgI4Q/AAAAGyH8AQAA2AjhDwhDEyZMkGVZJX7FxcWFusUA6enpevLJJ9WmTRt5PB55PB41bNhQnTt31p133qnvv//eX7ts2TL/+xgxYkSF9zJ79mxNmDBBEyZM0K5du8r1ml27dhX5jB0Oh2JiYtSqVSvde++92rdv3zn1VdjTpEmTzmkbhf3NmDHjnPoBUH25Qt0AgOotPz9fvXv31tq1awOW79+/X/v379fatWt11VVXqXXr1kHpZ/bs2XrrrbckSb169VKzZs1+1XaMMcrMzNS2bdu0bds2zZs3T1u3blV0dPSv2t5TTz0lSUpKStIDDzzwq7YBAOXBmT8gzA0fPlzGmICv48ePh7otvzlz5viD35VXXqmdO3cqJydHu3bt0ieffKI77rhDNWvWDHGXZ8cYI5/PpzVr1vjD3u7du7Vs2bLQNgYA5UD4A2xgxYoVuu6669SgQQNFRkaqXr16Gjx4sL755ht/zbZt2/xDhkOHDvUvHzdunH/5li1bJEmZmZmKjIyUZVnq0KFDqfvevn27//sePXqoWbNmioyMVFJSkvr376+//e1v+u1vf1vi62fPnq0OHTrI4/GoZcuWevnll2WMCag5cOCAHnjgAbVs2VJRUVGKjY3VJZdcohdeeEG5ubmSfhm6LTzrJ0mXX365/72dbXCzLEudO3fWRRdd5F+WmZnp/37FihUaNGiQWrRooVq1asnlcqlOnTrq27evZs+e7a8rHKottHv3bn9Pp5+VzMrK0nPPPadOnTqpRo0acrvdSkpK0o033qjs7Owi/Xm9Xj377LNq3ry5oqOj1aFDB33++edn9R4BVFMGQNgZP368kWQkmeHDh5daO3XqVGNZlr/+9K+IiAgzd+5cf22TJk2MJFOvXj3/sh49evjrX331VWOMMZ9//rl/2dixY0vd/z/+8Q9/rcPhMH369DETJkww8+fPN2lpaUXqly5d6q+vX79+sX3PmjXLX79jx44S6ySZHj16mKysLLNz584SaySZpUuXlvgeznytMcb4fD6zdu1aExMTYySZ2rVrm6NHj/pf8/LLL5e6v8L3cPq/5ZlfSUlJxhhjUlNTzcUXX1xi3bFjx4psq0GDBkXqIiMjzc6dO0v99wJQ/RH+gDBUWmA4PRDu3bvXuN1uI8lccskl5vvvvzc5OTlm3bp1pm7dukaSSUxMNHl5ecYYY0aMGOHfxtatW01mZqaJjIw0DofDSDJDhgwxxhjz+OOP++sWLlxYaq8ZGRmmZcuWxfbpdrvNiBEj/OHFmMDwJ8n85S9/MSdOnDBTpkzxL+vXr5+/vn///v7lw4YNM0eOHDHbt283bdu29S9//vnn/fXDhw8vV+A7XVnB0e12m0WLFgW8Zv369Wbx4sXmwIEDJicnx2RkZJi5c+f6X3PJJZcE1J8Z+E73xz/+0b/+ggsuMCtWrDAZGRlmx44d5plnnjHp6enGmMD/LmrUqGEWLlxojh8/bm655Rb/8okTJ5brPQOovhj2Baqx+fPnKycnR5L07bffqnXr1nK73erYsaMOHz4sqWDixcaNGyVJV1xxhf+1y5cv1+rVq5Wbm6uBAwfK7XZr+fLlkuQfInW73erRo0epPURHR2vNmjW67777lJiYGLAuJydHM2bM0MiRI4t9bfv27fXggw+qZs2aATN/C2fpZmVlaeHChZIKhmEnT56shIQEtWzZUhMmTPDXf/zxx6X2eK5ycnJ0/fXXa/369f5ljRs31ty5c9WrVy/FxcUpJiZGAwcO9K/funVrubf/0Ucf+b9/44031LNnT0VHR6tFixZ6/PHHFRMTU+Q1I0eOVN++fVWrVi3dfPPN/uXlneEMoPoi/AFhrrgJH4W3+Th48GC5tnHkyBFJRcNfYdi78sor1aVLFx0+fFjr1q3zT+Do3r27PB5PmduPj4/X5MmTtW/fPm3evFmvv/66unbt6l8/e/Zsf0g93ekzgE8POIXXuKWmpio/P1+SVKtWrYBb3Jx+vVx5P4fyKvyc9+/fr8GDB0uS0tLS9PTTT0uSfD6f+vTpo0mTJumHH35QVlZWkW0Ud51eSQ4cOOD/Pjk5uVyvKeuzA2BfhD+gGqtfv77/+7vuuqtISDSnZq3269fPX184gWH58uX+M3wpKSlKSUmRJP3pT3/yT6I4PSyWJC0tzf+9ZVlq06aN7rrrLi1fvtwfHL1eb7EzlCMiIgJee6batWvL5Sq4Y9WJEyd04sQJ/7rTz3Cd/jkUt51fq0GDBgFnJH/44QdJ0qZNm/Tdd9/5971p0ybl5+cHfBZnu59CmzdvLtdryvrsANgX4Q+oxq666iq53W5J0vTp0zVz5kydOHFCWVlZ2rBhg8aNG6du3boFvKYw0O3fv18rV65UvXr11Lp1a3/4mzNnTpHa0nzwwQdq27atJk2apM2bNys7O1vp6el65513/GfE6tatq3r16p31+/N4POrbt6+kgrNxDz74oI4ePar//Oc//rNwknTNNdf4v09ISPB//91338nn8531fgsdPHgw4GbKhcPahYFUkpxOp2JjY3XixAk99NBDJW6rsK8jR44UuWH09ddf7//+rrvu0pdffqmsrCzt2rVLzz77rDIyMn71ewBgQ6G51BDAuTib2b7Tpk0rcbaviplgMG/evID1Q4cONcYY/+SPwuXx8fHG6/WW2eubb75Z6mQJSWbq1Kn++tMnfJz53orrefv27f7JK8V9de3a1WRlZfnr//3vfxdbV5qyJnxIBTOZC2dO5+fnm4suuqhIzQUXXFDiPgcOHFjixJ1fM9t3+vTp5fpMAdgPZ/6Aau7uu+/WypUrNWTIECUmJsrlcql27dpKTk7W3XffrTfeeCOgPiUlJWDIsPCMn8fjUadOnfzLL7/8cjkcZR9CrrzySj3//PO6+uqr/fe8czqdqlu3rvr166fZs2dr1KhRv/r9tWzZUhs2bNC9996rFi1aKDIyUtHR0WrXrp0mTpyopUuXKioqyl9//fXXa/z48WrWrFnAGbpfw+VyKTExUYMGDdLixYs1YMAASQVn++bOnatrr71W8fHxqlmzpgYPHqwlS5aUuK0pU6Zo4MCBAWcmC8XHx2v16tWaOHGiOnTooNjYWEVGRqpp06YaOnRowPsDgLJYxpxxt1QAAABUW5z5AwAAsBHCHwAAgI0Q/gAAAGyE8AcAAGAjhD8AAAAbIfwBAADYCOEPAADARgh/AAAANkL4AwAAsBHCHwAAgI0Q/gAAAGyE8AcAAGAjhD8AAAAbIfwBAADYCOEPAADARgh/AAAANkL4AwAAsBHCXxW3Zs0aXXfddWratKncbrfq16+vrl27asyYMaFurUJlZmZqwoQJWrZsWdD3/fPPP2vChAnasGFD0PcNoGpYvXq1hg4dqsTEREVGRioxMVE33HCD1q5dG+rWqoxZs2Zp0qRJoW4DFYDwV4V98skn6tatm9LS0vT8889r4cKFmjx5srp37673338/1O1VqMzMTD311FMhC39PPfUU4Q+wqSlTpqh79+7au3evnn/+eS1atEgvvPCC9uzZo0svvVRvvPFGqFusEgh/1Ycr1A2gZM8//7zOO+88LViwQC7XL/9UN910k55//vkQdlZxjDHKzs4OdRsAbOrLL7/UAw88oP79++ujjz4qcqy97rrrdM8996h9+/bq1KlTUHvLzMxUdHR0UPcJe+DMXxV29OhR1alTJ+BgVMjhCPynsyxLEyZMKFLXrFkzjRgxwv/zjBkzZFmWPv/8c91+++2qXbu2YmJiNHDgQP33v/8NeG2vXr100UUXaeXKlbr00kvl8XjUqFEjPfHEE/J6vQG1qampuueee9SoUSNFRkaqefPmevzxx5WTk1Okz3vvvVevv/66WrduLbfbrbfeekt169aVJD311FOyLEuWZQX0fSafz6dnnnlGF154oTwej+Li4nTxxRdr8uTJAXU//vijbrnlFtWrV09ut1utW7fWa6+95l+/bNky/wH99ttv9++7uM8SQPUzceJEWZaladOmFTnWulwuTZ061V9XaMSIEWrWrFmRbU2YMEGWZQUsM8Zo6tSpateunTwej+Lj4zVkyJASj7crVqxQt27dFB0drTvuuEN33nmnateurczMzCL76927t9q0aVPq+1u/fr0GDBjgPwY2bNhQV199tfbu3XtWPfbq1UuffPKJdu/e7T9OnvleEUYMqqyRI0caSeaPf/yjWb16tcnNzS2xVpIZP358keVJSUlm+PDh/p+nT59uJJkmTZqYO+64w8yfP9+88cYbpl69eqZJkybm2LFj/tqUlBSTkJBgGjZsaF555RWzYMECc9999xlJZvTo0f66rKwsc/HFF5uYmBjz4osvmoULF5onnnjCuFwu079//yJ9NmrUyFx88cVm1qxZZsmSJWbDhg3ms88+M5LMnXfeaVatWmVWrVplduzYUeL7nThxonE6nWb8+PFm8eLF5rPPPjOTJk0yEyZM8Nds2bLF1KpVyyQnJ5uZM2eahQsXmjFjxhiHw+GvO3HihP8zGTdunH/fe/bsKXHfAKqH/Px8Ex0dbbp06VJqXefOnU2NGjWM1+s1xhgzfPhwk5SUVKRu/Pjx5sxfq7///e9NRESEGTNmjPnss8/MrFmzTKtWrUz9+vXNgQMH/HUpKSmmdu3apkmTJmbKlClm6dKlZvny5Wbjxo1GknnzzTcDtrtlyxYjybz22msl9p2enm4SEhJMx44dzQcffGCWL19u3n//fXP33XebrVu3nlWPW7ZsMd27dzcNGjTwHydXrVpV6ueGqovwV4UdOXLE9OjRw0gykkxERITp1q2bmThxojl58mRA7dmGv+uuuy6g7ssvvzSSzDPPPONflpKSYiSZOXPmBNT+/ve/Nw6Hw+zevdsYY8zrr79uJJkPPvggoO65554zkszChQsD+qxVq5ZJTU0NqD18+HCJ76E4AwYMMO3atSu1pl+/fqZx48bmxIkTAcvvvfdeExUV5e9h7dq1RpKZPn16ufYNoHo4cOCAkWRuuummUutuvPFGI8kcPnzYGFP+8Ldq1Sojybz00ksBdXv27DEej8eMHTvWv6zweLt48eIi201JSSlyvBs1apSpWbNmkd8Fp1u3bp2RZGbPnl1izdn0ePXVVxf7vhF+GPatwhISErRy5UqtXbtWzz77rAYNGqTt27fr0UcfVXJyso4cOfKrt33rrbcG/NytWzclJSVp6dKlActr1Kiha665JmDZLbfcIp/PpxUrVkiSlixZopiYGA0ZMiSgrnDYdvHixQHLe/furfj4+F/duyR17txZGzdu1D333KMFCxYoLS0tYH12drYWL16s6667TtHR0crPz/d/9e/fX9nZ2Vq9evU59QDAHowxknTWw5zz5s2TZVn63e9+F3AMatCggdq2bVtkglt8fLx69+5dZDv333+/NmzYoC+//FKSlJaWprffflvDhw9XbGxsifs///zzFR8fr4cfflivv/66tm7des49onog/IWBjh076uGHH9Y///lP/fzzz3rwwQe1a9euc5r00aBBg2KXHT16NGBZ/fr1S3xtYe3Ro0fVoEGDIgfGevXqyeVyFdlmYmLir+670KOPPqoXX3xRq1ev1lVXXaWEhAT16dNH69at8/eUn5+vKVOmKCIiIuCrf//+knRO4RlA+KtTp46io6O1c+fOUut27dolj8ejhISEs9r+wYMHZYxR/fr1ixyHVq9eXeQYVNKxcdCgQWrWrJn/euUZM2YoIyNDo0ePLnX/tWrV0vLly9WuXTs99thjatOmjRo2bKjx48crLy/vV/WI6oHZvmEmIiJC48eP18svv6zNmzf7l7vd7iKTKyQVCV6FDhw4UOyy888/P2DZwYMHS3xt4YEwISFBa9askTEmIAAeOnRI+fn5qlOnTsDrK+IiYZfLpYceekgPPfSQjh8/rkWLFumxxx5Tv379tGfPHsXHx8vpdOq2224r8QB53nnnnXMfAMKX0+lU7969NX/+fO3du1eNGzcuUrN371598803uvLKK/3LoqKiij3enhmU6tSpI8uytHLlSrnd7iL1Zy4r6djocDg0evRoPfbYY3rppZc0depU9enTRxdeeGGZ7zE5OVnvvfeejDH67rvvNGPGDD399NPyeDx65JFHzrpHVA+c+avC9u/fX+zy77//XpLUsGFD/7JmzZrpu+++C6hbsmSJ0tPTi93GO++8E/DzV199pd27d6tXr14By0+ePKmPP/44YNmsWbPkcDh02WWXSZL69Omj9PR0zZ49O6Bu5syZ/vVlKTzAZGVllVl7pri4OA0ZMkSjR49Wamqqdu3apejoaF1++eVav369Lr74YnXs2LHIV2F4PZd9AwhvjzzyiIwxuueee4rcxcDr9WrUqFHyer26//77/cubNWumQ4cOBfxxnJubqwULFgS8fsCAATLGaN++fcUeg5KTk8vd58iRIxUZGalbb71V27Zt07333ntW79OyLLVt21Yvv/yy4uLi9O233551j263m+NkNcGZvyqsX79+aty4sQYOHKhWrVrJ5/Npw4YNeumllxQbGxtwMLrtttv0xBNP6Mknn1RKSoq2bt2qV199VbVq1Sp22+vWrdPIkSM1dOhQ7dmzR48//rgaNWqke+65J6AuISFBo0aN0k8//aQLLrhAn376qd58802NGjVKTZs2lSQNGzZMr732moYPH65du3YpOTlZX3zxhf785z+rf//+uuKKK8p8rzVq1FBSUpLmzJmjPn36qHbt2qpTp06xt1OQpIEDB+qiiy5Sx44dVbduXe3evVuTJk1SUlKSWrZsKUmaPHmyevTooZ49e2rUqFFq1qyZTp48qR07dmju3LlasmSJJKlFixbyeDx655131Lp1a8XGxqphw4YB4RpA9dS9e3dNmjRJ999/v3r06KF7771XTZs21U8//aTXXntNq1at0oQJE9S3b1//a2688UY9+eSTuummm/S///u/ys7O1iuvvFIkPHbv3l1/+MMfdPvtt2vdunW67LLLFBMTo/379+uLL75QcnKyRo0aVa4+4+LiNGzYME2bNk1JSUkaOHBgma+ZN2+epk6dqmuvvVbNmzeXMUYffvihjh8/7n8/Z9NjcnKyPvzwQ02bNk0dOnSQw+FQx44dy/tRoyoJ2VQTlOn99983t9xyi2nZsqWJjY01ERERpmnTpua2224LmKZvjDE5OTlm7NixpkmTJsbj8ZiUlBSzYcOGEmf7Lly40Nx2220mLi7OeDwe079/f/Pjjz8GbDMlJcW0adPGLFu2zHTs2NG43W6TmJhoHnvsMZOXlxdQe/ToUXP33XebxMRE43K5TFJSknn00UdNdnZ2QJ3OuE3M6RYtWmTat29v3G63kRTQ95leeukl061bN1OnTh0TGRlpmjZtau68806za9eugLqdO3eaO+64wzRq1MhERESYunXrmm7dugXMajbGmHfffde0atXKREREnNWsYwDVw1dffWUGDx5s6tevbxwOh5FkoqKizCeffFJs/aeffmratWtnPB6Pad68uXn11VeLvdWLMcb8/e9/N126dDExMTHG4/GYFi1amGHDhpl169b5awqPt6VZtmyZkWSeffbZcr2nH374wdx8882mRYsWxuPxmFq1apnOnTubGTNm/KoeU1NTzZAhQ0xcXJyxLKvY94rwYBlzahoTbGHGjBm6/fbbtXbt2jL/YuvVq5eOHDkScG0hANjBzJkzNXz4cI0dO1bPPfdcqNuRJI0ZM0bTpk3Tnj17znryCXA6hn0BADjDsGHDtH//fj3yyCOKiYnRk08+GbJeVq9ere3bt2vq1Km66667CH44Z4Q/AACK8fDDD+vhhx8OdRvq2rWroqOjNWDAAD3zzDOhbgfVAMO+AAAANsKtXgAAAGyE8AcAAGAjhD8AAAAbIfwBAADYCOEPAADARgh/AAAANkL4AwAAsBHCHwAAgI0Q/gAAAGyE8AcAAGAjhD8AAAAbIfwBAADYCOEPAADARgh/AAAANkL4AwAAsBHCHwAAgI24Qt1AOMvNzdPS1ZuUejxdzZvWV+e2F8iyrFC3BQC2lZ/v1Yqvt+jgkeNq1CBBPTq2lsPBeQ7gdIS/X2naO/M17i//UOrxdP+yVi0a640/3aOendqEsDMAsKf35q7QQ3/6u/YfPuZfltSorl6dcJcG9O4Uws6AqsUyxphQNxFuJk3/WA8+87ciyx0OS06HQyvem6hL218Ygs4AwJ5mfbxctz74lyLLC0dj5r05Tv0v7xjstoAqifB3ltJOZqp+l+HKzsktdr3T4VC3Dq204r2JQe4MAOwpLy9fjbrdrsOpacWutyxLLZo20PbF07g0BxATPs7ahwtWKaeE4CdJXp9PK9du1a69B4PYFQDY16IvN5YY/CTJGKMdu/fr643bg9gVUHUR/s7S/sPH5HCW/bHtP3SszBoAwLk7/Rq/Uus4LgOSCH9nLdLlktfrK7MusV58ELoBACTWLd/xluMyUIDZvmdp1fofyqzp1qGVmjWuH4RuAABXdG+rurVrlnnNX+e2F5S5reycXH2ydJ0OHD6mxHq11b9XB0W5Iyu6ZSCkCH9nIfX4Sc1ZtKbMur7d21V+MwAASVJEhEuTnhhZ6mzfyU+MLHOyx98++Fz/8+fpOn4yQ5ZlyRijuBox+su4O3T7kCsqpXcgFBj2PQu79x1WfhlDvs5yXA8IAKhYt1yToncnjSkytJvUqK4+fuPxMm/z8ta/l2jko6/q+MkMSQWTRCTp+MkM3fHwFL390dLKaRwIAc78nYVaNaLLrDHGlKsOAFCxbhp4mYb2767lawqe8NE4MUHdO5T9hI+8vHyNfW5GqTVjn5uhmwdeJpfLWYEdA6FB+DsLzZs2ULvfnKfvvt8lXwm3RzRGGnxltyB3BgCQJKfTqd7dLj6r1yxbs1mHjp4otebA4eNavmaz+nRvey7tAVUCY5Rn6ZmHfiej4oOfZVn6/Y191bRh3SB3BQD4tQ6nlh78zrYOqOoIf2fp6ss7auaLDyomOkqWpAiXUw6HJcuS7rzhCr064a5QtwgAOAtNEuuUq44/7FFd8Hi3Xyk9I0v/nP+l/rP7gOJqxmjIVd24vQsAhCGfz6fzL79bu/YdUnG/Ei3LUvMm9fXjktd5PByqBcIfAMD2Fqz4Vlff+X8yxgRc0+2wLFmWpfnTx6tvj3ahaxCoQAz7AgBsr99ll+izGeP1m5ZNApa3uaCpFr71FMEP1Qpn/gAAOMUYo03bdmv/oVQ1rF9bF12QxFAvqh3CHwAAgI0w7AsAAGAjhD8AAAAbIfwBAKoNY4xOpmcqLy8/1K0AVRbhDwAQ9k6mZ2rC5HdVv/Mw1Wx7szxthmrIPc/qm007Qt0aUOUw4QMAENbSTmbqspsf0+Ztu+X1+fzLXU6HZFma+8Y4XZlySQg7BKoWzvwBAMLaU1PeKxL8JCnf65PX69PND7yorOycEHUHVD2EPwBA2MrOydWb7y0sEvwKGWN0PC1D//z0yyB3BlRdhD8AQNjau/+oTmZklVoT4XJq07bdQeoIqPoIfwCAsOWJiiyzxhhTrjrALgh/AICw1bB+bbVtfZ4cpTyCLd/r06ArugSxK6BqI/wBAMKWZVl64t4b5CvhxhVOp0O9u16sDsnnB7kzoOoi/AEAwtrgK7tp0riRcjgsORwOOZ0OuVxOSdKl7S7Qv157OMQdAlUL9/kDAFQL+w4c1fR/Lda2nftUI8ajoVd1U69Lk2WVMiQM2BHhDwAAwEYY9gUAALARwh8AAICNEP4AAABshPAHAABgI4Q/AAAAGyH8AQAA2AjhDwAAwEYIfwCAsGKMUdrJTGVkZoe6FSAsEf4AAGHB6/Xq1ZmfqGXvu1Wr3c2KTb5R3YaM1ZzP14S6NSCs8IQPAECV5/V6ddP9L+rfn30lSSr8zeVwWPL5jJ4bO1xj77o+hB0C4YMzfwCAKm/Wxyv0r/lfyZhfgp8k+XwFPzz8/Fv6fseeEHUHhBfCHwCgynv17U/kcFglrnc5Hfp/7y4IYkdA+CL8AQCqvM3bf/Kf5StOvtenDd/vDGJHQPgi/AEAqjyPO7LU9ZZlKcbjDlI3QHgj/AEAqrzBV3aTy1nyryxjjK7v1zWIHQHhi/AHAKjyHrh9oJxOpxxW0ev+nE6HGjdI0M0DLwtBZ0D4IfwBAKq81uc30dw3xykmOkqWJJfTKZfLKUlqmlhXS/7xjKIZ9gXKhfv8AQDCxsn0TL3z8XJ9vfFHuZxOXZlyia7p09kfBAGUjfAHAABgIwz7AgAA2AjhDwAAwEYIfwAAADZC+AMAALARwh8AAICNEP4AAABshPAHAABgI4Q/AAAAGyH8AQAA2AjhDwAAwEYIfwAAADZC+AMAALARwh8AAICNEP4AAABshPAHAABgI4Q/AAAAGyH8AQAA2AjhDwAAwEYIfwAAADZC+AMAALARwh8AAICNEP4AAABshPAHAABgI4Q/AAAAGyH8AQAA2AjhDwAAwEYIfwAAADZC+AMAALARwh8AAICNEP4AAABsxBXqBuwiPSNLi77cqPTMbLVq3kgdks+XZVmhbgsAANgM4a8SGGP09cbtWr1+u5xOS9t3/ay/f7BIGVk5/pq2rc/T9OfuU/s2zUPYKQAAsBvLGGNC3UR18uPOn3XDfc9rw9adcjgs+XzFf7xOh0NRUZFa+9GLan1+kyB3CQAA7Ipr/irQwSPH1fOmR7Vp225JKjH4SZLX51N2Tq4mTH43WO0BAAAQ/irSa29/oiOpafJ6feWq93p9+veCVTqZnlnJnQEAABQg/FWgGf9eIq+vfMGvkNfrU+qJ9ErqCAAAIBDhrwKlHj951q9xuZxKiKtRCd0AAAAURfirQEmN6p3V7VucTodu6N9dsTGeSuwKAADgF4S/CnTXzf3KXet0OhTjcWvCfTdXYkcAAACBCH8VaOSNv1X735wnp7Psj7VTckt9+cFzanlewyB0BgAAUID7/FWwtJOZGjPx73r7o2XKyc2TJMXVjNHo2/qra/tWysrOUavmjXXRhUkh7hQAANgR4a+SHE9L16Ztu+VyOtW+TXNFuSND3RIAAADhDwAAwE645g8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANgI4Q8AAMBGCH8AAAA24gp1AwAAAFVNdnaOftrzsyxLSmraSJGRkaFuqcIQ/gAAAE7Jzc3V27M+0pLlq+X1eiVJTqdTvXp20e3Dhsrlcoa4w3NnGWNMqJsAAAAItfz8fD3x9MvauWtPseubNE7Uc888LIcjvK+aC+/uAQAAKsjylWtKDH6StGfvfn348YIgdlQ5CH8AAACSPv5kUZk1CxauCEInlYtr/gAAACSlHjtRZs3J9Az5fD75fEbfrN+kr1Z/q5PpGUpsUFe9U7qqRfOkIHR6bgh/AAAAklxOl/Ly8susSzuZrmdfmKZdP+2TZVkyxuiHbf/R4qVf6bdX9NTttw2RZVlB6PjXYdgXAABAUusLW5RZk5AQrylT39JPe/dLkgrnzfp8PknSwkUrNX/BskrrsSIQ/gAAACQNv21wmTW9enbRlu9/9Ie94nz86eJS14ca4Q8AAEBS/Xp1dPfIW0pc36NbR0VGRsjhKH1I9/jxNO37+UBFt1dhuOYPAADglF6XXaqWLZrpvX/N03ebf5DP51PTxg01aGBfdepwsWbPXSjJklT6bZLz871B6ffXIPwBAACcplGjBhpz/8hi1zU/r2mZQ7pud6QSE+tVRmsVgmFfAACAckpuc6Hq16tT4tCvw2Gpd0pXRbndQe6s/Ah/AAAA5eRwOPTQfXcqKioq4DFvllXw1fy8prpx6IAQdlg2nu0LAABwlo4cPab5C5dr5ZdfKysrW3Xr1FbfPj3Vp1dXRUZGhrq9UhH+AAAAbIRhXwAAABsh/AEAANgI4Q8AAMBGCH8AAAA2QvgDAACwEcIfAACAjRD+AAAAbITwBwAAYCOEPwAAABsh/AEAANiIK9QNACi/I0ePafnK1dp/4LBioqPVtUt7XXhBc1mWFerWAABhgmf7AmHi9b/O0rIVq4ssb/ObC/Q/94+UxxMVgq4AAOGGYV8gDLw85W/FBj9J2vr9dr36+swgdwQACFeEP6CK+2Hbf7Rm7cYS1xsjfbN+s/bu2x/ErgAA4YrwB1Rxc+Z9Xq66b77dXMmdAACqA8IfUMXt3LW3XHW5eXmV3AkAoDog/AFVnNsdWa66Jo0TK7kTAEB1QPgDqriOlySrrDu5REZGqOMlycFpCACqkJPpGfr3R5/pr9Pf04LPlys/Pz/ULVV53OoFqOIOHjqiMY/8Sfn53hJr7rtnhLpdeon27P1ZX6/7Tjk5uWrSOFFdOrVVZGT5zhwCQDjx+XyaMu0trVqzPmC5w+HQDYP769qBvw1RZ1Uf4Q8IAxu+26qXJv9VeXmBf9E6HA7dMXyoul/aQVOmvaVvN2yRw+GQZVnyer2KjvZo9F23qUP7i0LUOQBUjhcnval1324qcf0tN16ja66+IogdhQ/CHxAm0tJOauny1dryw48yxqhN6wvUp1c3xcZGa+IL07R56zb5fEX/d3Y4HHpq3P1qef55Qemz8JDCU0cAVJZDh47ovv95utQal8upmX99SQ4HV7idice7AWGiZs0aGjSwrwYN7BuwfMd/duu7zT+U+toPP16ohx+6qzLb06bN2zRv/hJt3rpdPp9P5zdvqgH9+6hzx7YEQQAVau78JWXW5Od7tfab79SlU7vKbyjMEP6AMLfq6/VyOhzy+nzFrvf5fFq/YYuyc3IU5XZXSg+ffrZUM2d9JMuy/Gf+fvzPbr085e9qm9xaD4+5i7++qwhjjJT3nZSzQjLZkquFFNVPliMm1K0B5ZaaerxcdfsPHKrcRsIUR2MgzGVlZknlOLGWk5NbKfv/ac/PmjnrI0m/DPmebuOm7/XS5L8Wuw7BZXwnpGN3F3xlvi9lfSydfFE6PEAme1mo2wPKrW7d2uWqa1C/biV3Ep4If0CYS2xQr9hr/U7n8UQpNia6Uvb/+eIvyhzW/Wb9Zm3esr1S9o/yMcZIx8ZIeVtOLfFKypdkJGVLJx6XyS354nmgKhk0oG+ZNS6Xk1tglYDwB4S5y3p2lsNRcvhyOBzq06ubnE5npex/x393l+us3sLFKytl/yinvG+k/C0qCH3FsaSMt4LZEfCrxcfVUo9uHUutGTr4arlcXN1WHMIfEOZq1ayh4bcOllR0hq3D4VCD+nV07TWVd7+riHIeXHftLt9j6lBJspdKKu0PAK+U+5WMqZzLA4CKdu/dw9SnV7dij3s3DR2oQdzmpUREYqAa+O0VPRUXV1P/nv2Zdv+0T1LBUz969eyiGwZfXWlDvpLUof1F2r5jZ5l15X1MHcrHP3Ej62PJu1eyakqefpI7RZYVUcwLslUwxFvqViWTI1n8WyE8/P6OmzT8d9dr4eIvdDT1mBo3TFRKz86c8SsD9/kDqhFjjI6mHldubq4SascHJXClpZ3U6AfGK6+URypZlqXrB/XT0Ov7V3o/dmCMT0p7Vsqeq4KzeV4VDOT4JNf5Uvwrshzxga/JeFtKn6ZSA6AVL9WdJ8tiUAiozvg/HKhGLMtSnYR4NUysH7QzbTVr1tBD999Zak9RUW5d0bt7UPqxhcxZp4Kf9Ms1fKdu9ZO/Uzo+ruhroq5W6Yd8hxQ9mOAH2AD/lwM4Z+3bttGTj/1RUVG/3Eew8Dqc2NhojXt4tOLjaoWqvWrFmHwpY1YpFV4p71uZvB0BSy1nbanG2FM/nXnod0iuC6WYWyqyVQBVFMO+ACpMbm6eVn+9Xlt/2CEZo9atzlfXLu0VGck1ZBXF5P0opQ4ro8ohxY6SFfO7oq/PWSVlzCi4XlCSrBpS9PVSzHBZlqfC+wVQ9RD+ACCMmLwfpNTby6hySLF/kBUzvOTt+E5IJldyxMuyuDgesBP+jweAcOJKkhQlKbuUIp8UcVGpm7EcDMMDdsU1fwAQRizLI0Vfo5IP307J2VSKuCSYbQEII4Q/AAg3sXdLrt+o4KHOp9/g1iFZsVLcxDIfuQfAvrjmDwDCkDE5UtY8KesjyftzQejz9Jc8g2U5eZg9gJIR/gAAAGyEYV8AAAAbYbYvAFRzJneTlPWhlPe9ZEVJ7hTJM6jgxs8AbIdhXwCoxkz6G1LGdP3yDGCpYGKIR4qfLCuiTQi7AxAKDPsCQDVlspeeCn7SL8FPknySyZKOPSRjskLRGoAQIvwBQHWVMUslH+Z9kkmTsj8PZkcAqgDCHwBUQ8bkS/mbJflKqXJIud8GqyUAVQQTPgCgWirv5dxc9o2q4/iJNK38Yq0OHDqs6GiPunZur+bnNQ11W9UO4Q8AqiHLipBxtZLyt6vks38+KaJtMNsCSvTZ5ys0850PZYyRw2HJGGnuJ4t1Sbs2um/0CEW53aFusdpg2BcAqqvom1Vy8Dv1KLiofsHsCCjW6q/Xa8bb/5LP55MxRl6vTz5fwX+7GzZu1bQ33glxh9UL4Q8AqquovpLnxlM/OE9b4ZAUKcW9IMsRE4LGgF8YY/Svj+arpMdR+4zRmrUb9PP+g8FtrBoj/AFANWVZllTjfinuFcndQ3LUk5xNpOhbpTrvyopsF+oWAR08dER79x1QaXcddjgsrf3mu+A1Vc1xzR8AVGOWZUnuTgVfQBWUk5NbZo1lOZSdXXYdyoczfwAAIGTq1q0tl6v0c1Fer1dNGicGqaPqj/AHAABCJtrjUc/uneRwFB9JLEuKjYlWpw7JQe6s+iL8AQCAkLp56ADVSYgvEgAdDoccDodG3z1MERERIequ+rGMKe0SSwBAODLen6XsJZIxUlRvWa5GoW4JKNXJkxn68OMFWrpslbJzcmRZltq1/Y2uH9RPLVs0C3V71QrhDwCqEeNNlY7dJ3n/E7jCmSTFvyLLWS80jQHllJ/vVXp6hqKi3IqK4sbOlYHwBwDVhPFlSEeulUx6CRUeqc5sWc6awWwLQBXDrV6AMuz+aZ/mzPtce/bul9vt1mU9Oqlv7x4Ft9AAqpL0/1dK8JOkLCn9FanWuKC1BKDq4cwfUAKv16uJL0zT5q3bi6yLinLrqXEPKqlpwxB0BhTPHLpCMhllVLll1V8WjHYAVFHM9gWKkZ+frzGP/LnY4CdJ2dk5GvfUizqRdjLInQGlMFnlKMoRf/MD9kb4A4rx9qyPdODg4VJr8vLy9cn8pUHqCCiPyHLUuLhkAbA5wh9whszMLC1a+lW5apd/saaSuwHOQlSvsmvc3Sq9DQBVW9hN+DDGSDlLpMx/SvnbJUVI7hQp5kZZrhahbg/VwLYf/yuv11uu2vI8kxIImtg/StmLJeWVUOCUYh8IYkMAqqKwOvNnjJHS/iSdGCflbSq4vsWkSdmfSkdHyOR8EeoWUQ14vb5y19arm1CJnQBnx3LWlhJmSlatYlbGSrX/KsvF81EBuwuvM3/Zn0rZn5z64fRf0F5JlnR8nEzdObIcxRz4gHI6r1ljWZZVroviB1zVOwgdAeVnuZpJ9T6TyVkjZS+U5JPcfWRF9Qh1awCqiPAKf5nvS7IkFfdL2UjKlbI+lWJuDm5fqFYSaser4yXJWvftplID4AUtm6l71w5B7AwoP8vdRXJ3CXUbAKqgsBn2NSZfyv9RxQe/0+RtDko/qN5G3n6j6terU+L6nt076YlH/iin0xnErgAAOHdhc5NnY7zSoZ4qPfw5pKg+smo9Hay2UI1lZmbp8yVfaPGyr5Saelwx0dHq1OFiXTeonxJqx4W6PQAAfpWwCX+SZI7dJ+V+o8Dr/c5Q8wlZnv5B6wkAAKCi+Hw+bdz0vb5Zv1l5eflKatJQl/XorNjYmArbR3iFv5w10vEHSljrkBzxUp1/ybKigtkWAADAOUtNPa6JL07Tnr375XQ4ZFRwpxOXy6nRd92mSzu3r5D9hM01f9KpC5hrPKiCSR+F11pZBV9WLSluEsEPAACEHZ/Ppz+/MFX7fj4oSfL6fPL5fDLGKC8vX5Nfm6HtP+6skH2FVfiTJCv6BinhXSl6qBTRTorsLNUYI9X5p6yI80PdHgAACFPp6Rn6ccdO7f5pn3y+8t/ztSJ8u2GL9u47UOJ+LcvSnHmLKmRf4XWrl1MsV5JU4/5QtwEAAKqB4yfS9I93Z2vVmm/9N/pPSIjX9df0U+9eXYPyPOx1326Sw+EoMfz5fD59u2GzfD6fHI5zO3cXluEPAACgIqSdTNeTT7+sI0ePBQSvo0eP6c3p7+n4iTQNvvbKSu8jNze3zIcLGGOUn+9VZOS5hb+wG/YFAACoKB/PW1Qk+J3uXx/N1+EjqZXeR5PGDcusqZMQr8jIiHPeV8jCnzE+mdyNMtmLZHK/LbiPHwAAQJD4fD4tXvZVqdf3WZa0fOWaSu/l8ssuLXV42bIsXfXblArZV0jCn8n5QjoyRDp2t3TiCenYaOnI9TLZS0LRDgAAsKHMrGxlZWWXUWXp0KGjld5LXFxN/eGOmySpyDV9lmXpN61bql/fyypkX0G/5s/kfCkdH6siT+rwHZJOPC6j/5MVdUWw2wIAADYT5Y6U0+nwT/IojiUpJjY6KP30uuxSJSTEa868z7V5y3ZJUu34OPXr21P9+/WSy1UxsS2o4c8YI6U9p1If0XZykoy7lyyLuSgAAKDyuFwudenUXqu/Xl/i0K/X51OPrh2C1lNymwuV3OZC5ebmKj/fK48nqsJnGwd32Dd3jeQ7XHqN7+ipR7gBAABUruuu6Sun01lswLIsS5e0a6MWzZOC3ldkZKSioz2VcpuZ4Ia/jH+Ur853pHL7AAAAUMEs28fH3qP4+FqSCq63KwxcXbtcovtH3x7K9ipFUJ/taw72kZRZdmHsaFkxv6v0fgAAAKSCmb8bN32vn/b8rMjISHVof5Hq1U0IdVuVIsjhL0VSbtmFEZ1k1X6l0vsBAACwm+AO+zriyleXt07GV44zhAAAADgrwQ1/niHlLDSSyrrvDgAAAM5WcMNf9K2SapRdZ9WQ8o/IePeX+Zw7AAAAlF9Qr/mTJJN/WDo6SKXe60/WL+tdLaWY38uK6hmE7gAAAKq3oD/ezXLVlWo+XkbVacEwf4d0YqxM1txK7QsAAMAOgn7mr5DJWSNlzJDyNpxaEiEpXyWfEYyU6s6T5SjHsDEAAACKFbLwV8j40iRfmnT0Fkl5pVRaUo0xsqIHB6s1AACAaifow75nshw1T31XWvCTJKfk3VvZ7QAAAFRrIQ9/kiRHbDmKjGSVpw4AAAAlcVXWho3JkbKXSd7dkhUtuVNkuZoUW2s54mQi2kt5GyX5StiiV4rqU1ntAkBYM/l7pMz3pOxFksmSXE0kz2DJM1CWFRHq9gBUIZVyzZ/JXiGl/Z9k0lWQL30FX+7fSrUek2W5i74m91vp2B9VfPizJPcVsuKeruhWAaDKMiZLyv9JslySs5ksy1l8Xe5G6dgDKrh8xntq6albZkV2lOJekmVFBqdpAFVehYc/k7teOjZaxc/adUju3rLi/q/412Yvk9KekUyGAkJjVD+p5qPFhkYAqG6ML0M6OUnKXiD/9dCOOlLMbZJnqCzL+qXW5EmHB0nmhEr84znmTlmxdwahcwDhoELDn8nfK6WOOBXeSpHwrixXs+K3YbKl7OWSd5dkxUjuXrJcjSuqRQCo0oz3SMHdD8zJ4gs8Q2XVfOiX+uzPpRNPlr5RK06qO1eWVWlX+gAIIxV2JDDeQ1LqyLKDn5wF16TEjix2rWVFSZ5+FdUWAIQNY3Kk1DtKDn6SlPVPGc/VsiIuLPg573sVHMrzS9nwccl3WHImVmC3AMJVxc32zfhH6QcsP+vUtYAAgADZCwpCWqksKWvOaT+7VPrjMgsVf70gAPupkPBnjJGy56nkmbqn80pOhnEBoIiseeUoMoH3PHVfql8meRTHkpxJkqPuOTYHoLqooDN/OQW3FigXV8EEDgBAIG9ZZ/1OsWr+8n1Ee8l1gUo+s2ekmGEBk0QA2FsFhT+3ZHnKV1rjf3g+LwAUx1nOs3NRff3fWpYlxb1w2ohK4WH9VBiMGSFFXVVRHQKoBipkwodlWTJRA6SsD1Xq8EPs/bKir6mIXQJA9eMZIOVtKr3G0UBydw9YZDnrySTMLLixfs5SyZcuuZpJnkGyIs6vtHYBhKcKu9WL8R6Sjo6QTJqKDYDuq2TFlXE7AgCwsYLZvn+Q8neo2GuorWgp4T1Z5T1DCADFqLDZvpaznlT7DSniojPWRErRv5NqPVZRuwKAasmy3FL8q5L7cgUeni0psqtUZw7BD8A5q5zHu+XvOvWXa6QU2UGWI6aidwEA1ZrxHjo1BGxJEcmEPgAVplLCHwAAAKqmirvJMwAAAKo8wh8AAICNEP4AAABshPAHAABgI4Q/AAAAGyH8AQAA2AjhDwAAwEYIfwAAADZC+AMAALARwh8AAICNEP4AAABshPAHAABgI4Q/AAAAGyH8AQAA2AjhDwAAwEYIfwAAADZC+AMAALARwh8AAICNEP4AAABshPAHAABgI4Q/AAAAG/n/xK81E2k7eiIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 800x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "pca = PCA(n_components=2)\n",
    "support_pca = pca.fit_transform(support_features)\n",
    "query_pca = pca.fit_transform(query_features)\n",
    "\n",
    "fig, ax = plt.subplots(1, 2, figsize=(8, 5))\n",
    "ax[0].scatter(support_pca[:,0], support_pca[:,1], c=support_targets)\n",
    "ax[0].set_title(\"Support set\")\n",
    "ax[0].axis('off')\n",
    "ax[1].scatter(query_pca[:,0], query_pca[:,1], c=query_targets)\n",
    "ax[1].set_title(\"Query set\")\n",
    "ax[1].axis('off')\n",
    "plt.suptitle(\"Few Shot Batch\", weight='bold')\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define DCCA neural network used in SEEDV paper"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define CCA methods and classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cca_metric_derivative(H1, H2):\n",
    "    r1 = 1e-3\n",
    "    r2 = 1e-3\n",
    "    eps = 1e-9\n",
    "    # transform the matrix: to be consistent with the original paper\n",
    "    H1 = H1.T\n",
    "    H2 = H2.T\n",
    "    # o1 and o2 are feature dimensions\n",
    "    # m is sample number\n",
    "    o1 = o2 = H1.shape[0]\n",
    "    m = H1.shape[1]\n",
    "\n",
    "    # calculate parameters\n",
    "    H1bar = H1 - H1.mean(axis=1).reshape([-1,1])\n",
    "    H2bar = H2 - H2.mean(axis=1).reshape([-1,1])\n",
    "\n",
    "    SigmaHat12 = (1.0 / (m - 1)) * np.matmul(H1bar, H2bar.T)\n",
    "    SigmaHat11 = (1.0 / (m - 1)) * np.matmul(H1bar, H1bar.T) + r1 * np.eye(o1)\n",
    "    SigmaHat22 = (1.0 / (m - 1)) * np.matmul(H2bar, H2bar.T) + r2 * np.eye(o2)\n",
    "\n",
    "    # eigenvalue and eigenvector decomposition\n",
    "    [D1, V1] = np.linalg.eigh(SigmaHat11)\n",
    "    [D2, V2] = np.linalg.eigh(SigmaHat22)\n",
    "\n",
    "    # remove eighvalues and eigenvectors smaller than 0\n",
    "    posInd1 = np.where(D1 > 0)[0]\n",
    "    D1 = D1[posInd1]\n",
    "    V1 = V1[:, posInd1]\n",
    "\n",
    "    posInd2 = np.where(D2 > 0)[0]\n",
    "    D2 = D2[posInd2]\n",
    "    V2 = V2[:, posInd2]\n",
    "\n",
    "    # calculate matrxi T\n",
    "    SigmaHat11RootInv = np.matmul(np.matmul(V1, np.diag(D1 ** -0.5)), V1.T)\n",
    "    SigmaHat22RootInv = np.matmul(np.matmul(V2, np.diag(D2 ** -0.5)), V2.T)\n",
    "    Tval = np.matmul(np.matmul(SigmaHat11RootInv,SigmaHat12), SigmaHat22RootInv)\n",
    "    # By default, we will use all the singular values\n",
    "    tmp = np.matmul(Tval.T, Tval)\n",
    "    corr = np.sqrt(np.trace(tmp))\n",
    "    cca_loss = -1 * corr\n",
    "\n",
    "    # calculate the derivative of H1 and H2\n",
    "    U_t, D_t, V_prime_t = np.linalg.svd(Tval)\n",
    "    Delta12 = SigmaHat11RootInv @ U_t @ V_prime_t @ SigmaHat22RootInv\n",
    "    Delta11 = SigmaHat11RootInv @ U_t @ np.diag(D_t) @ U_t.T @ SigmaHat11RootInv\n",
    "    Delta22 = SigmaHat22RootInv @ U_t @ np.diag(D_t) @ U_t.T @ SigmaHat22RootInv\n",
    "    Delta11 = -0.5 * Delta11\n",
    "    Delta22 = -0.5 * Delta22\n",
    "\n",
    "    DerivativeH1 = ( 1.0 / (m - 1)) * (2 * (Delta11 @ H1bar) + Delta12 @ H2bar)\n",
    "    DerivativeH2 = ( 1.0 / (m - 1)) * (2 * (Delta22 @ H2bar) + Delta12 @ H1bar)\n",
    "\n",
    "    return cca_loss, DerivativeH1.T, DerivativeH2.T\n",
    "\n",
    "class cca_loss():\n",
    "    def __init__(self, outdim_size, use_all_singular_values, device):\n",
    "        self.outdim_size = outdim_size\n",
    "        self.use_all_singular_values = use_all_singular_values\n",
    "        self.device = device\n",
    "\n",
    "    def loss(self, H1, H2):\n",
    "        r1 = 1e-3\n",
    "        r2 = 1e-3\n",
    "        eps = 1e-9\n",
    "\n",
    "        H1, H2 = H1.t(), H2.t()\n",
    "        o1 = o2 = H1.size(0)\n",
    "\n",
    "        m = H1.size(1)\n",
    "\n",
    "        H1bar = H1 - H1.mean(dim=1).unsqueeze(dim=1)\n",
    "        H2bar = H2 - H2.mean(dim=1).unsqueeze(dim=1)\n",
    "\n",
    "        SigmaHat12 = (1.0 / (m - 1)) * torch.matmul(H1bar, H2bar.t())\n",
    "        SigmaHat11 = (1.0 / (m - 1)) * torch.matmul(H1bar,\n",
    "                                                    H1bar.t()) + r1 * torch.eye(o1, device=self.device)\n",
    "        SigmaHat22 = (1.0 / (m - 1)) * torch.matmul(H2bar,\n",
    "                                                    H2bar.t()) + r2 * torch.eye(o2, device=self.device)\n",
    "\n",
    "        # Calculating the root inverse of covariance matrices by using eigen decomposition\n",
    "        breakpoint()\n",
    "        [D1, V1] = torch.symeig(SigmaHat11, eigenvectors=True)\n",
    "        [D2, V2] = torch.symeig(SigmaHat22, eigenvectors=True)\n",
    "\n",
    "        # Added to increase stability\n",
    "        posInd1 = torch.gt(D1, eps).nonzero()[:, 0]\n",
    "        D1 = D1[posInd1]\n",
    "        V1 = V1[:, posInd1]\n",
    "        posInd2 = torch.gt(D2, eps).nonzero()[:, 0]\n",
    "        D2 = D2[posInd2]\n",
    "        V2 = V2[:, posInd2]\n",
    "\n",
    "        SigmaHat11RootInv = torch.matmul(\n",
    "            torch.matmul(V1, torch.diag(D1 ** -0.5)), V1.t())\n",
    "        SigmaHat22RootInv = torch.matmul(\n",
    "            torch.matmul(V2, torch.diag(D2 ** -0.5)), V2.t())\n",
    "\n",
    "        Tval = torch.matmul(torch.matmul(SigmaHat11RootInv,\n",
    "                                         SigmaHat12), SigmaHat22RootInv)\n",
    "\n",
    "        if self.use_all_singular_values:\n",
    "            # all singular values are used to calculate the correlation\n",
    "            tmp = torch.matmul(Tval.t(), Tval)\n",
    "            corr = torch.trace(torch.sqrt(tmp))\n",
    "            # assert torch.isnan(corr).item() == 0\n",
    "        else:\n",
    "            # just the top self.outdim_size singular values are used\n",
    "            trace_TT = torch.matmul(Tval.t(), Tval)\n",
    "            trace_TT = torch.add(trace_TT, (torch.eye(trace_TT.shape[0])*r1).to(self.device)) # regularization for more stability\n",
    "            U, V = torch.symeig(trace_TT, eigenvectors=True)\n",
    "            U = torch.where(U>eps, U, (torch.ones(U.shape)*eps).to(self.device))\n",
    "            U = U.topk(self.outdim_size)[0]\n",
    "            corr = torch.sum(torch.sqrt(U))\n",
    "        return -corr"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define DCCA network layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformLayers(nn.Module):\n",
    "    def __init__(self, input_size, layer_sizes):\n",
    "        super(TransformLayers, self).__init__()\n",
    "        layers = []\n",
    "        layer_sizes = [input_size] + layer_sizes\n",
    "        for l_id in range(len(layer_sizes) - 1):\n",
    "            if l_id == len(layer_sizes) - 2:\n",
    "                layers.append(nn.Sequential(\n",
    "                    #nn.BatchNorm1d(num_features=layer_sizes[l_id], affine=False),\n",
    "                    nn.Linear(layer_sizes[l_id], layer_sizes[l_id+1]),\n",
    "                    ))\n",
    "            else:\n",
    "                layers.append(nn.Sequential(\n",
    "                    nn.Linear(layer_sizes[l_id], layer_sizes[l_id+1]),\n",
    "                    nn.Sigmoid(),\n",
    "                    #nn.BatchNorm1d(num_features=layer_sizes[l_id+1], affine=False),\n",
    "                    ))\n",
    "        self.layers = nn.ModuleList(layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x)\n",
    "        return x\n",
    "\n",
    "class AttentionFusion(nn.Module):\n",
    "    def __init__(self, output_dim):\n",
    "        super(AttentionFusion, self).__init__()\n",
    "        self.output_dim = output_dim\n",
    "        self.attention_weights = nn.Parameter(torch.randn(self.output_dim, requires_grad=True))\n",
    "    def forward(self, x1, x2):\n",
    "        # calculate weigths for all input samples\n",
    "        row, _ = x1.shape\n",
    "        fused_tensor = torch.empty_like(x1)\n",
    "        alpha = []\n",
    "        for i in range(row):\n",
    "            tmp1 = torch.dot(x1[i,:], self.attention_weights)\n",
    "            tmp2 = torch.dot(x2[i,:], self.attention_weights)\n",
    "            alpha_1 = torch.exp(tmp1) / (torch.exp(tmp1) + torch.exp(tmp2))\n",
    "            alpha_2 = 1 - alpha_1\n",
    "            alpha.append((alpha_1.detach().cpu().numpy(), alpha_2.detach().cpu().numpy()))\n",
    "            fused_tensor[i, :] = alpha_1 * x1[i,:] + alpha_2 * x2[i, :]\n",
    "        return fused_tensor, alpha\n",
    "\n",
    "class DCCA_AM(nn.Module):\n",
    "    def __init__(self, input_size1, input_size2, layer_sizes1, layer_sizes2, outdim_size, categories, device):\n",
    "        super(DCCA_AM, self).__init__()\n",
    "        self.input_dim_split = input_size1\n",
    "        self.outdim_size = outdim_size\n",
    "        self.categories = categories\n",
    "        # self.use_all_singular_values = use_all_singular_values\n",
    "        self.device = device\n",
    "\n",
    "        self.model1 = TransformLayers(input_size1, layer_sizes1).to(self.device)\n",
    "        self.model2 = TransformLayers(input_size2, layer_sizes2).to(self.device)\n",
    "\n",
    "        # convert generator object to list for deepcopy(model) to work\n",
    "        self.model1_parameters = list(self.model1.parameters()) \n",
    "        self.model2_parameters = list(self.model2.parameters())\n",
    "\n",
    "        self.classification = nn.Linear(self.outdim_size, self.categories)\n",
    "\n",
    "        self.attention_fusion = AttentionFusion(outdim_size)\n",
    "    def forward(self, x):\n",
    "        x1 = x[:, :self.input_dim_split]\n",
    "        x2 = x[:, self.input_dim_split:]\n",
    "        # forward process: returns negative of cca loss and predicted labels\n",
    "        output1 = self.model1(x1)\n",
    "        output2 = self.model2(x2)\n",
    "        # cca_loss_val = self.loss(output1, output2)\n",
    "        cca_loss, partial_h1, partial_h2 = cca_metric_derivative(output1.detach().cpu().numpy(), output2.detach().cpu().numpy())\n",
    "        fused_tensor, alpha = self.attention_fusion(output1, output2)\n",
    "        out = self.classification(fused_tensor)\n",
    "        return out, cca_loss, output1, output2, partial_h1, partial_h2, fused_tensor.detach().cpu().data, alpha"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define meta learning model and methods"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define baseline ProtoNet model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ProtoNet(pl.LightningModule):\n",
    "\n",
    "    def __init__(self, lr, model_args):\n",
    "        \"\"\"\n",
    "        Inputs\n",
    "            proto_dim - Dimensionality of prototype feature space\n",
    "            lr - Learning rate of Adam optimizer\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.save_hyperparameters()\n",
    "        eeg_input_dim, eye_input_dim, layer_sizes1, layer_sizes2, \\\n",
    "            output_dim, num_emotions, device = model_args\n",
    "        self.model = DCCA_AM(eeg_input_dim, eye_input_dim, layer_sizes1, layer_sizes2, output_dim, num_emotions, device).to(device)\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        optimizer = optim.AdamW(self.parameters(), lr=self.hparams.lr)\n",
    "        scheduler = optim.lr_scheduler.MultiStepLR(\n",
    "            optimizer, milestones=[140, 180], gamma=0.1)\n",
    "        return [optimizer], [scheduler]\n",
    "\n",
    "    @staticmethod\n",
    "    def calculate_prototypes(features, targets):\n",
    "        # Given a stack of features vectors and labels, return class prototypes\n",
    "        # features - shape [N, proto_dim], targets - shape [N]\n",
    "        features = features[0]\n",
    "        # targets = targets.reshape((1,-1))[0] \n",
    "        # already RESHAPED EARLIER in dataset_from_labels call\n",
    "        classes, _ = torch.unique(targets).sort() # Determine which classes we have\n",
    "        prototypes = []\n",
    "        # print(\"targets:\", targets)\n",
    "        for c in classes:\n",
    "            # print(\"c:\", c)\n",
    "            # print(features[torch.where(targets == c)[0]])\n",
    "            # maybe use for target in targets loop\n",
    "            p = features[torch.where(targets == c)[0]].mean(dim=0)  # Average class feature vectors\n",
    "            prototypes.append(p)\n",
    "        prototypes = torch.stack(prototypes, dim=0)\n",
    "        # Return the 'classes' tensor to know which prototype belongs to which class\n",
    "        return prototypes, classes\n",
    "\n",
    "    def classify_feats(self, prototypes, classes, feats, targets):\n",
    "        # Classify new examples with prototypes and return classification error\n",
    "        dist = torch.pow(prototypes[None, :] - feats[:, None], 2).sum(dim=2)  # Squared euclidean distance\n",
    "        preds = F.log_softmax(-dist, dim=1)\n",
    "        labels = (classes[None, :] == targets[:, None]).long().argmax(dim=-1)\n",
    "        acc = (preds.argmax(dim=1) == labels).float().mean()\n",
    "        return preds, labels, acc\n",
    "\n",
    "    def calculate_loss(self, batch, mode):\n",
    "        # Determine training loss for a given support and query set \n",
    "        features, targets = batch\n",
    "        outputs = self.model(features)  # Encode all images of support and query set\n",
    "        support_feats, query_feats, support_targets, query_targets = split_query_support(outputs, targets)\n",
    "        prototypes, classes = ProtoNet.calculate_prototypes(support_feats, support_targets)\n",
    "        preds, labels, acc = self.classify_feats(prototypes, classes, query_feats, query_targets)\n",
    "        loss = F.cross_entropy(preds, labels)\n",
    "\n",
    "        self.log(f\"{mode}_loss\", loss)\n",
    "        self.log(f\"{mode}_acc\", acc)\n",
    "        return loss\n",
    "\n",
    "    def training_step(self, batch, batch_idx):\n",
    "        return self.calculate_loss(batch, mode=\"train\")\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        _ = self.calculate_loss(batch, mode=\"val\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define ProtoMAML model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ProtoMAML(pl.LightningModule):\n",
    "    \n",
    "    def __init__(self, lr, lr_inner, lr_output, num_inner_steps, model_args):\n",
    "        \"\"\"\n",
    "        Inputs\n",
    "            eeg_input_dim - Dimensionality of prototype feature space\n",
    "            lr - Learning rate of the outer loop Adam optimizer\n",
    "            lr_inner - Learning rate of the inner loop SGD optimizer\n",
    "            lr_output - Learning rate for the output layer in the inner loop\n",
    "            num_inner_steps - Number of inner loop updates to perform\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.save_hyperparameters()\n",
    "        eeg_input_dim, eye_input_dim, layer_sizes1, layer_sizes2, \\\n",
    "            output_dim, num_emotions, device = model_args\n",
    "        self.model = DCCA_AM(eeg_input_dim, eye_input_dim, layer_sizes1, layer_sizes2, output_dim, num_emotions, device).to(device)\n",
    "        \n",
    "    def configure_optimizers(self):\n",
    "        optimizer = optim.AdamW(self.parameters(), lr=self.hparams.lr)\n",
    "        scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[140,180], gamma=0.1)\n",
    "        return [optimizer], [scheduler]\n",
    "        \n",
    "    def run_model(self, local_model, output_weight, output_bias, features, labels):\n",
    "        # Execute a model with given output layer weights and inputs\n",
    "        dcca_out = local_model(features)\n",
    "        out = dcca_out[0]\n",
    "        cca_loss = dcca_out[1]\n",
    "        # get only first element of tuple in feats\n",
    "        preds = F.linear(out, output_weight, output_bias)\n",
    "        # loss = F.cross_entropy(preds, labels.reshape((-1,1))[0]) \n",
    "        # already RESHAPED EARLIER in dataset_from_labels calls\n",
    "        loss = 0.7 * cca_loss + 1.0 * F.cross_entropy(preds, labels)\n",
    "        acc = (preds.argmax(dim=1) == labels).float()\n",
    "        return loss, preds, acc\n",
    "        \n",
    "    def adapt_few_shot(self, support_features, support_targets):\n",
    "        # Determine prototype initialization\n",
    "        support_feats = self.model(support_features)\n",
    "        prototypes, classes = ProtoNet.calculate_prototypes(support_feats, support_targets)\n",
    "        support_labels = (classes[None,:] == support_targets[:,None]).long().argmax(dim=-1)\n",
    "        # Create inner-loop model and optimizer\n",
    "        local_model = deepcopy(self.model)\n",
    "        local_model.train()\n",
    "        local_optim = optim.SGD(local_model.parameters(), lr=self.hparams.lr_inner)\n",
    "        local_optim.zero_grad()\n",
    "        # Create output layer weights with prototype-based initialization\n",
    "        init_weight = 2 * prototypes\n",
    "        init_bias = -torch.norm(prototypes, dim=1)**2\n",
    "        output_weight = init_weight.detach().requires_grad_()\n",
    "        output_bias = init_bias.detach().requires_grad_()\n",
    "        \n",
    "        # Optimize inner loop model on support set\n",
    "        for _ in range(self.hparams.num_inner_steps):\n",
    "            # Determine loss on the support set\n",
    "            loss, _, _ = self.run_model(local_model, output_weight, output_bias, support_features, support_labels)\n",
    "            # Calculate gradients and perform inner loop update\n",
    "            loss.backward()\n",
    "            local_optim.step()\n",
    "            # Update output layer via SGD\n",
    "            output_weight.data -= self.hparams.lr_output * output_weight.grad\n",
    "            output_bias.data -= self.hparams.lr_output * output_bias.grad\n",
    "            # Reset gradients\n",
    "            local_optim.zero_grad()\n",
    "            output_weight.grad.fill_(0)\n",
    "            output_bias.grad.fill_(0)\n",
    "            \n",
    "        # Re-attach computation graph of prototypes\n",
    "        output_weight = (output_weight - init_weight).detach() + init_weight\n",
    "        output_bias = (output_bias - init_bias).detach() + init_bias\n",
    "        \n",
    "        return local_model, output_weight, output_bias, classes\n",
    "        \n",
    "    def outer_loop(self, batch, mode=\"train\"):\n",
    "        accuracies = []\n",
    "        losses = []\n",
    "        self.model.zero_grad()\n",
    "        \n",
    "        # Determine gradients for batch of tasks\n",
    "        for task_batch in batch:\n",
    "            features, targets = task_batch\n",
    "            support_features, query_features, support_targets, query_targets = split_query_support(features, targets)\n",
    "            # Perform inner loop adaptation\n",
    "            local_model, output_weight, output_bias, classes = self.adapt_few_shot(support_features, support_targets)\n",
    "            # Determine loss of query set\n",
    "            query_labels = (classes[None,:] == query_targets[:,None]).long().argmax(dim=-1)\n",
    "            loss, preds, acc = self.run_model(local_model, output_weight, output_bias, query_features, query_labels)\n",
    "            # Calculate gradients for query set loss\n",
    "            if mode == \"train\":\n",
    "                loss.backward()\n",
    "\n",
    "                for p_global, p_local in zip(self.model.parameters(), local_model.parameters()):\n",
    "                    p_global.grad += p_local.grad  # First-order approx. -> add gradients of finetuned and base model\n",
    "            \n",
    "            accuracies.append(acc.mean().detach())\n",
    "            losses.append(loss.detach())\n",
    "        \n",
    "        # Perform update of base model\n",
    "        if mode == \"train\":\n",
    "            opt = self.optimizers()\n",
    "            opt.step()\n",
    "            opt.zero_grad()\n",
    "        \n",
    "        self.log(f\"{mode}_loss\", sum(losses) / len(losses))\n",
    "        self.log(f\"{mode}_acc\", sum(accuracies) / len(accuracies))\n",
    "    \n",
    "    def training_step(self, batch, batch_idx):\n",
    "        self.outer_loop(batch, mode=\"train\")\n",
    "        return None  # Returning None means we skip the default training optimizer steps by PyTorch Lightning\n",
    "        \n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        # Validation requires to finetune a model, hence we need to enable gradients\n",
    "        torch.set_grad_enabled(True)\n",
    "        self.outer_loop(batch, mode=\"val\")\n",
    "        torch.set_grad_enabled(False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define task batch sampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TaskBatchSampler(object):\n",
    "    \n",
    "    def __init__(self, dataset_targets, batch_size, N_way, K_shot, include_query=False, shuffle=True):\n",
    "        \"\"\"\n",
    "        Inputs:\n",
    "            dataset_targets - PyTorch tensor of the labels of the data elements.\n",
    "            batch_size - Number of tasks to aggregate in a batch\n",
    "            N_way - Number of classes to sample per batch.\n",
    "            K_shot - Number of examples to sample per class in the batch.\n",
    "            include_query - If True, returns batch of size N_way*K_shot*2, which \n",
    "                            can be split into support and query set. Simplifies\n",
    "                            the implementation of sampling the same classes but \n",
    "                            distinct examples for support and query set.\n",
    "            shuffle - If True, examples and classes are newly shuffled in each\n",
    "                      iteration (for training)\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.batch_sampler = FewShotBatchSampler(dataset_targets, N_way, K_shot, include_query, shuffle)\n",
    "        self.task_batch_size = batch_size\n",
    "        self.local_batch_size = self.batch_sampler.batch_size\n",
    "        \n",
    "    def __iter__(self):\n",
    "        # Aggregate multiple batches before returning the indices\n",
    "        batch_list = []\n",
    "        for batch_idx, batch in enumerate(self.batch_sampler):\n",
    "            batch_list.extend(batch)\n",
    "            if (batch_idx+1) % self.task_batch_size == 0:\n",
    "                yield batch_list\n",
    "                batch_list = []\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.batch_sampler)//self.task_batch_size\n",
    "    \n",
    "    def get_collate_fn(self):\n",
    "        # Returns a collate function that converts one big tensor into a list of task-specific tensors\n",
    "        def collate_fn(item_list):\n",
    "            features = torch.stack([feat for feat, target in item_list], dim=0)\n",
    "            targets = torch.stack([target for feat, target in item_list], dim=0)\n",
    "            features = features.chunk(self.task_batch_size, dim=0)\n",
    "            targets = targets.chunk(self.task_batch_size, dim=0)\n",
    "            return list(zip(features, targets))\n",
    "        return collate_fn"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define model training and testing methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model_class, train_loader, val_loader, **kwargs):\n",
    "    trainer = pl.Trainer(default_root_dir=os.path.join(CHECKPOINT_PATH, model_class.__name__),\n",
    "                         accelerator=\"gpu\" if str(device).startswith(\"cuda\") else \"cpu\",\n",
    "                         devices=1,\n",
    "                         max_epochs=50,\n",
    "                         callbacks=[ModelCheckpoint(save_weights_only=True, mode=\"max\", monitor=\"val_acc\", every_n_epochs=1),\n",
    "                                    LearningRateMonitor(\"epoch\")],\n",
    "                         enable_progress_bar=False,\n",
    "                         log_every_n_steps=12)\n",
    "    trainer.logger._default_hp_metric = None\n",
    "\n",
    "    # Check whether pretrained model exists. If yes, load it and skip training\n",
    "    pretrained_filename = os.path.join(\n",
    "        CHECKPOINT_PATH, model_class.__name__ + \".ckpt\")\n",
    "    if os.path.isfile(pretrained_filename):\n",
    "        print(f\"Found pretrained model at {pretrained_filename}, loading...\")\n",
    "        # Automatically loads the model with the saved hyperparameters\n",
    "        model = model_class.load_from_checkpoint(pretrained_filename)\n",
    "    else:\n",
    "        pl.seed_everything(42)  # To be reproducable\n",
    "        model = model_class(**kwargs)\n",
    "        trainer.fit(model, train_loader, val_loader)\n",
    "        model = model_class.load_from_checkpoint(\n",
    "            trainer.checkpoint_callback.best_model_path)  # Load best checkpoint after training\n",
    "\n",
    "    return model\n",
    "\n",
    "def test_protomaml(model, dataset, k_shot=4):\n",
    "    pl.seed_everything(42)\n",
    "    model = model.to(device)\n",
    "    num_classes = dataset.targets.unique().shape[0]\n",
    "    exmps_per_class = dataset.targets.shape[0]//num_classes\n",
    "    \n",
    "    # Data loader for full test set as query set\n",
    "    full_dataloader = data.DataLoader(dataset, \n",
    "                                      batch_size=128, \n",
    "                                      num_workers=32, \n",
    "                                      shuffle=False, \n",
    "                                      drop_last=False)\n",
    "    # Data loader for sampling support sets\n",
    "    sampler = FewShotBatchSampler(dataset.targets, \n",
    "                                  include_query=False,\n",
    "                                  N_way=num_classes,\n",
    "                                  K_shot=k_shot,\n",
    "                                  shuffle=False,\n",
    "                                  shuffle_once=False)\n",
    "    sample_dataloader = data.DataLoader(dataset, \n",
    "                                        batch_sampler=sampler,\n",
    "                                        num_workers=32)\n",
    "    \n",
    "    # We iterate through the full dataset in two manners. First, to select the k-shot batch. \n",
    "    # Second, the evaluate the model on all other examples\n",
    "    accuracies = []\n",
    "    for (support_features, support_targets), support_indices in tqdm(zip(sample_dataloader, sampler), \"Performing few-shot finetuning\"):\n",
    "        support_features = support_features.to(device)\n",
    "        support_targets = support_targets.to(device)\n",
    "        try:\n",
    "            # Finetune new model on support set\n",
    "            local_model, output_weight, output_bias, classes = model.adapt_few_shot(support_features, support_targets)\n",
    "            with torch.no_grad():  # No gradients for query set needed\n",
    "                local_model.eval()\n",
    "                batch_acc = torch.zeros((0,), dtype=torch.float32, device=device)\n",
    "                # Evaluate all examples in test dataset\n",
    "                for query_features, query_targets in full_dataloader:\n",
    "                    query_features = query_features.to(device)\n",
    "                    query_targets = query_targets.to(device)\n",
    "                    query_labels = (classes[None,:] == query_targets[:,None]).long().argmax(dim=-1)\n",
    "                    _, _, acc = model.run_model(local_model, output_weight, output_bias, query_features, query_labels)\n",
    "                    batch_acc = torch.cat([batch_acc, acc.detach()], dim=0)\n",
    "                # Exclude support set elements\n",
    "                for s_idx in support_indices:\n",
    "                    batch_acc[s_idx] = 0\n",
    "                batch_acc = batch_acc.sum().item() / (batch_acc.shape[0] - len(support_indices))\n",
    "                accuracies.append(batch_acc)\n",
    "        except np.linalg.LinAlgError:\n",
    "            print(\"NaN layer encountered\")\n",
    "    return mean(accuracies), stdev(accuracies)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train meta learning model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training constant (same as for ProtoNet)\n",
    "N_WAY = 5\n",
    "K_SHOT = 10\n",
    "\n",
    "# Training set\n",
    "train_protomaml_sampler = TaskBatchSampler(train_set.targets, \n",
    "                                           include_query=True,\n",
    "                                           N_way=N_WAY,\n",
    "                                           K_shot=K_SHOT,\n",
    "                                           batch_size=16)\n",
    "train_protomaml_loader = data.DataLoader(train_set, \n",
    "                                         batch_sampler=train_protomaml_sampler,\n",
    "                                         collate_fn=train_protomaml_sampler.get_collate_fn(),\n",
    "                                         num_workers=32)\n",
    "\n",
    "# Validation set\n",
    "val_protomaml_sampler = TaskBatchSampler(val_set.targets, \n",
    "                                         include_query=True,\n",
    "                                         N_way=N_WAY,\n",
    "                                         K_shot=K_SHOT,\n",
    "                                         batch_size=1,  # We do not update the parameters, hence the batch size is irrelevant here\n",
    "                                         shuffle=False)\n",
    "val_protomaml_loader = data.DataLoader(val_set, \n",
    "                                       batch_sampler=val_protomaml_sampler,\n",
    "                                       collate_fn=val_protomaml_sampler.get_collate_fn(),\n",
    "                                       num_workers=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "Global seed set to 42\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name  | Type    | Params\n",
      "----------------------------------\n",
      "0 | model | DCCA_AM | 90.4 K\n",
      "----------------------------------\n",
      "90.4 K    Trainable params\n",
      "0         Non-trainable params\n",
      "90.4 K    Total params\n",
      "0.362     Total estimated model params size (MB)\n",
      "/home/chanel/miniconda3/envs/pl-env/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py:136: UserWarning: `training_step` returned `None`. If this was on purpose, ignore this warning...\n",
      "  self.warning_cache.warn(\"`training_step` returned `None`. If this was on purpose, ignore this warning...\")\n",
      "`Trainer.fit` stopped: `max_epochs=50` reached.\n"
     ]
    }
   ],
   "source": [
    "EEG_INPUT_DIM = 310\n",
    "EYE_INPUT_DIM = 33\n",
    "OUTPUT_DIM = 12\n",
    "LAYER_SIZES = [200, 50, OUTPUT_DIM]\n",
    "NUM_EMOTIONS = N_WAY\n",
    "\n",
    "model_ckpt_path = os.path.join(os.curdir, \"saved_models\", \"seed-v\", \"protomaml_model.ckpt\")\n",
    "\n",
    "if os.path.exists(model_ckpt_path):\n",
    "  protomaml_model = ProtoMAML.load_from_checkpoint(model_ckpt_path)\n",
    "else:\n",
    "  protomaml_model = train_model(ProtoMAML, \n",
    "                                lr=1e-3, \n",
    "                                lr_inner=0.1,\n",
    "                                lr_output=0.1,\n",
    "                                num_inner_steps=1,\n",
    "                                model_args = (EEG_INPUT_DIM, EYE_INPUT_DIM, LAYER_SIZES,\n",
    "                                  LAYER_SIZES, OUTPUT_DIM, NUM_EMOTIONS, device),\n",
    "                                train_loader=train_protomaml_loader, \n",
    "                                val_loader=val_protomaml_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-7c090cfb1e0ab01\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-7c090cfb1e0ab01\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          const port = 6015;\n",
       "          if (port) {\n",
       "            url.port = port;\n",
       "          }\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Opens tensorboard in notebook. Adjust the path to your CHECKPOINT_PATH if needed\n",
    "%tensorboard --logdir saved_models/seed-v/ProtoMAML/lightning_logs/version_11"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test meta learning model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Increase number of inner steps, if necessary, for further improvement in testing phase\n",
    "protomaml_model.hparams.num_inner_steps = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 42\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58b1b3004ca7468db3c31125e0bf7fd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Performing few-shot finetuning: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 42\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f1cf377d2414120a3ffe93d58015e1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Performing few-shot finetuning: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 42\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ab9ac595c4d48218c70c047e6324e2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Performing few-shot finetuning: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for k=5: 45.72% (+-12.21%)\n",
      "Accuracy for k=10: 44.60% (+-8.27%)\n",
      "Accuracy for k=15: 45.63% (+-8.79%)\n"
     ]
    }
   ],
   "source": [
    "protomaml_result_file = os.path.join(CHECKPOINT_PATH, \"protomaml_fewshot.json\")\n",
    "\n",
    "# Perform same experiments as for ProtoNet\n",
    "protomaml_accuracies = dict()\n",
    "for k in [5, 10, 15]:\n",
    "    protomaml_accuracies[k] = test_protomaml(protomaml_model, test_set, k_shot=k)\n",
    "# Export results\n",
    "with open(protomaml_result_file, 'w') as f:\n",
    "    json.dump(protomaml_accuracies, f, indent=4)\n",
    "\n",
    "for k in protomaml_accuracies:\n",
    "    print(f\"Accuracy for k={k}: {100.0*protomaml_accuracies[k][0]:4.2f}% (+-{100.0*protomaml_accuracies[k][1]:4.2f}%)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_few_shot(acc_dict, name, color=None, ax=None):\n",
    "    sns.set()\n",
    "    if ax is None:\n",
    "        fig, ax = plt.subplots(1,1,figsize=(5,3))\n",
    "    ks = sorted(list(acc_dict.keys()))\n",
    "    mean_accs = [acc_dict[k][0] for k in ks]\n",
    "    std_accs = [acc_dict[k][1] for k in ks]\n",
    "    ax.plot(ks, mean_accs, marker='o', markeredgecolor='k', markersize=6, label=name, color=color)\n",
    "    ax.fill_between(ks, [m-s for m,s in zip(mean_accs, std_accs)], [m+s for m,s in zip(mean_accs, std_accs)], alpha=0.2, color=color)\n",
    "    ax.set_xticks(ks)\n",
    "    ax.set_xlim([ks[0]-1, ks[-1]+1])\n",
    "    ax.set_xlabel(\"Number of shots per class\", weight='bold')\n",
    "    ax.set_ylabel(\"Accuracy\", weight='bold')\n",
    "    if len(ax.get_title()) == 0:\n",
    "        ax.set_title(\"Few-Shot Performance \" + name, weight='bold')\n",
    "    else:\n",
    "        ax.set_title(ax.get_title() + \" and \" + name, weight='bold')\n",
    "    ax.legend()\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdsAAAFACAYAAAD029a0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABiMElEQVR4nO3dd2BT5f7H8Xd2Ogi0lI2sIntPkSlTBFTwqoggIHJRhvLTXmV4nSjorSiggAoKyhIUUFQ2COKgLBVkV/YslDbdSZPz+yPNIeluaUgL39e9leTM56RpPnme85znaBRFURBCCCGEz2j9XQAhhBDiVidhK4QQQviYhK0QQgjhYxK2QgghhI9J2AohhBA+JmErhBBC+JiErRBCCOFjErZCCCGEj0nYCiGEED6m93cBRMHNmjWLDz/8MMf5pUqVYvfu3TexRHmz2+18+eWXrFmzhpMnT2K32yldujTlypWjTp06DBw4kBYtWgBw9uxZunXrBkCbNm348ssvi7QsO3fuJCoqCoDu3btTv379fK1Xt27dLNMMBgNhYWG0bduWp59+mpo1axZpWT1dvXqVd955h19//ZXY2FgcDgfdunVj9uzZPtungK5du3Lu3DmvaTqdjtDQUFq0aMFTTz1FkyZNfLb/lStXqvsfOnQoFoul0NvK/NnRtm1bvvjiC69l/vrrLx5++OEs00wmk9e0lJQU2rdvT1JSEgAajYZNmzZRtWrVLPvN/LezaNEiWrdu7TXtqaee4ueff1afjxw5koiICMD1GkycOBHwzWfCzSA1W3FTjB8/nnfeeYeDBw+SnJyM3W7nypUrHDp0iG+//ZZ9+/bdtLJERUXx4Ycf8uGHH3Lo0KEb2pbdbufChQusXr2aAQMGcODAgSIqZVZvvfUW3377LTExMTgcDp/tR+TN4XAQExPD+vXreeyxx/jpp598tq9Vq1ap71er1Vqk2965cydHjx71mpY5fHOyfv16NWgBFEVh9erV+Vo3c1iePHmSHTt25GvdkkrCtoTr378/R44c8fopbrXaAwcOsGnTJgAaNGjA2rVr2b9/P9u3b2fBggUMHjyYsmXL+rmUBbN582aOHDnCpk2b1FpNcnIykZGRRbofu91Oeno6AH///XeW/RdlrdZzXyJ7X3zxBUeOHGHHjh3cc889AKSnp/PWW2/luW5KSoqvi1coixYtUh9fuXKFdevW5Wu9VatWqY81Go06LT/D7W/evJmLFy96leFWH6ZfwvY2sGvXLsaMGUP79u1p1KgR7dq1Y9y4cV61sH/++Ye6detSt25dnn32WXX6+++/r04/duwY4PrQaNSoEXXr1mXAgAF57v/kyZPq44YNG1KrVi2MRiMVKlSgXbt2/Pe//+XBBx/Mcf3du3fz+OOP07RpU7p27cr06dOx2+1eyyQmJvLBBx/Qt29fmjZtSpMmTejTpw/vv/8+iYmJ6nJ169b1akabOHGienwrV67M81gyu+OOO3jqqafU53/++afX/E2bNjFixAjatm1Lw4YN6dixIy+++KLXawIwYcIEtRybNm3ilVde4e6776Zx48Z899131K1b12udbt26UbduXWbNmgW4ahUrVqxg4MCBtGzZkkaNGnHPPfcwceJETp06le99Xbx4kZUrV6rzZ86cySeffEKXLl1o2rQpw4cP58SJEyQkJPDKK6/Qtm1b7rrrLsaPH8/Vq1e99jNnzhwGDRpEhw4daNKkCY0bN6Zbt25MmjSJs2fPei07ZMgQdZ/79u3jpZdeom3btrRq1YqnnnqK06dPZ3ntf/75Z55++mn1fd22bVsGDx6sniJwvy4rV67k8ccfp1WrVjRq1IiuXbvy+uuvExMTk/cvOAflypVj7Nix6vPTp08TGxsLuJqd3ccSHR3NqFGjaNGiBX369FGXP3XqFJMmTaJr1640atSIFi1aMHDgQJYvX66Gzs6dO6lbt67X8bh/73Xr1lVfw4L87j25m3vXrFmj1piXLVuG3W7PtinY0/nz59VyVa1alfbt2wOuU0C7du3Kdd2qVauSnp7O0qVLAUhKSlKDO6/9lmRyzvYWt2TJEt544w2vb42xsbFs2LCBrVu3MmvWLO655x5q1apFpUqVuHDhglfN2PMPJyoqijvvvJN9+/apYdeuXbs8y1CpUiX18YoVKzh+/Dht2rShSZMmtGzZkpCQkBzXPXr0KEOHDlVrXOfOnePjjz8mKCiIUaNGqcczaNAgTpw44bXu8ePHOX78OOvXr2fp0qW57udG5PSNPDIykk8//dRr2uXLl/n222/ZuHEjX3zxBY0bN86y3ssvv8y1a9cKtP/x48dnqZGcP3+elStXsm7dOhYsWEDTpk0LvK+lS5eqIQLw66+/MmLECMqVK8cff/yhTl+7di0JCQnMnz9fnbZu3ToOHz7stb2zZ89y9uxZtm/fzvfff0+ZMmWy7HPUqFHEx8erz92humbNGnQ6HQAffPABc+bM8VovLi6OXbt2cfDgQdq0aYOiKLzwwgv88MMPXsudO3eOJUuWsHHjRpYtW1boD/j81MQef/xx9fV1H+sff/zB8OHDSU5OVpez2+3s27ePffv28csvv/DBBx/kuwyF/d336NGD77//npiYGL755hsGDx7MV199BcCgQYN49913c9zvqlWrcDqdANx3331Ur15dbQZetWoVbdq0yXFd97aXL1/OmDFjWL16NYmJiZQpU4bevXtn+Zu5VUjNtoRbtWqV+k3X/TNhwgQALl26xNSpU1EUhYYNG/Ljjz+yf/9+vvnmG0JDQ7Hb7fz3v/9Vg8wdnFevXiU6OprU1FT++usvtFrX22Tnzp1e/wLcfffdeZaxefPmtGzZUn2+b98+Pv74Y7W2PXr0aK8mJU9xcXE88cQTREVF8dFHH3kdt9usWbPUoO3QoQPbt29n+/btatlOnDjBzJkzAThy5IhXjWTq1Klq83t+aumZnTlzhnnz5qnPmzVrBsD+/fvVD42OHTuyZcsW9u/fz4IFCzAYDCQnJ/Paa69lu02Hw8GcOXPYt28fa9eupXfv3hw5coQqVaqoy7jLPG7cONatW6d+2FapUoWVK1eye/duRo4cCbiatydPnpyvfWVuzk9KSmL+/Pns2rVLbS4/d+4cR48eZdGiRezYsUP9MrVjxw6v2uK4ceP47rvviIqK4u+//+bXX39VX+OYmBjWrFmTbZkqVKjA2rVr2b59O+Hh4QBER0ezf/9+wHVawh20Wq2Wl19+md9//53ff/+dDz/8UF1nw4YNatAOGDCAHTt2sH//ft577z21DLkFSm6uXLni1UJSvXp1QkNDsyxXrlw5Vq9ezZ9//qmWefLkyWrQjho1it27d7Ny5Ur1dXT/Ptu2bcuRI0e8gst9+uDIkSNUrVr1hn73er2egQMHArB48WLWrVvH5cuXCQgI4F//+leux//tt9+qj++77z569OiBwWBQy+/5RSKz/v37ExwcTGxsLN9//z2LFy8G4JFHHsnSCetWImF7C9u+fTs2mw1wne+77777aNy4MQ899JBaW4mJiVFrH5611KioKP744w/sdjv33HMPRqNRreW6m4+MRqNXiOZEq9Xy2WefMXr0aO644w6veQ6Hg82bNzN27Fj1m7Kn0NBQXnjhBUqXLk337t3V2oFn79DNmzerj//zn/9QoUIFKlSowIsvvqhO37JlS57lLAh3c1737t3VEAgICOD5558HUM9Rg6tm1rVrVxo3bsywYcPUVoEDBw541Rrdhg0bRteuXQkMDKRWrVoEBATkWhbP4x82bBgNGzakVKlSjB8/Xn29jh07lm1TbF776tatGx06dMBisXh96Hft2pXWrVtTrlw5tRc5eP9eypQpw/vvv0+fPn1o1qwZd999t1dT/fHjx7M9nvHjx1OrVi0qVKhA586ds2zb87V98MEHGTJkCCEhIYSEhNCjRw86duwIwMaNG9XlVq5cSYcOHWjcuDEvvPCCOt2z92t+PPHEE9StW5f27durnaJ0Op36BTezV199lfr162M2m6lbty6nTp1SjzskJITnnnuOUqVK0bBhQ4YNG6aul9/364387gEGDhyIwWDgzJkzTJkyBYB+/fpRunTpHPe5e/dutXm6Ro0a1K9fn9KlS6tfbpOTk1m/fn2O6wcFBdG/f38A3nnnHaKjo9HpdDz22GP5OuaSSsK2hMuug9S0adMA17fv/HA3c3nWUnft2qWGa8eOHWnatCmxsbHs379fDZcWLVpgNpsB7/OA7h/3+UQAs9nMc889x6ZNm9i4cSPTpk1TPxTBVRM8c+ZMlrJVr14dvf762Y7AwEAA9UsE4HWu0LP25/k48/nEoqLX66lYsSL3338/33zzjdosnN/9xcXFZZnWqFGjApXBc1+VK1fOUja37N4Pee2rWrVq6mP37xq8z60ZjUb1cVpaGuA6d/3EE0+wdetWYmJispxj91w2M3fNFPAKf/fynsdRp06dHMuen99BcnKy13spv3Q6HWFhYfTs2ZMlS5bQtWvXbJdr2LCh13PPslesWFFtFgfv1zS/758b+d0DhIWFce+99wLX34uPP/54rvv0/MLUvHlzDh06xKFDh2jQoEG2y2Rn8ODBaDQadZ/dunXzKv+tSM7Z3sLCwsLUx48++ihvvPFGlmUURVF7EoaFhVGnTh2OHj1KVFSU2iTYpk0bLl++zK5du5g7d676wZmfJmRwfaCZTCb1g6VatWpUq1aN/v3707NnT/VbclxcHNWrV/da19005eYuq6eyZcty6dIlwFX7qVevnvrYc5nctlFQmzdvzvVcn+f+XnjhBf79739nWcbztffkGWr54bmv8+fPq48dDodX87zn+yG/+/L8opOf6W4//PCDenlSv379mDx5MiEhIXz55ZdqDSo/+8zu9fE8jsyXrXjyfF2mT5/u1UHJLaffQU6++OIL2rZtm+/lM7cUeJb94sWLOBwO9e/Cs9NYfnvn38jv3u2JJ55Qm/TbtGmj/v1kJyUlxev88KpVq7xO6bjt2rWLs2fP5vg3UqNGDTp27Mj27dsBV/je6qRmewvr1KmTWutYuXIlq1evJiEhgdTUVA4dOsT777+vnrNxczclx8TEsHv3bsqWLUt4eLjahOjZbOXZ7Dxt2rQsNexx48YBrg4hvXr1Ys6cORw4cIDk5GRSU1PZunWr+oGg1+sLPSCEZ60iMjKSS5cucfnyZa/LcDyX8eyUc+zYMZ9c7tK9e3f18bx589i6dSvJyckkJSXxxx9/MGXKFK9zxzfC89gWLFjAoUOHSExMZMaMGWrNoXbt2l61VF/zrLGZTCbMZjOHDx/O9zWcuenRo4cakN9++y2LFy/m2rVrxMXFsWXLFrVpuEePHuo67733HlFRUaSlpZGQkMDOnTuZOHEir7/++g2XpyCqV6+u1tyvXbvGzJkzSUhI4NChQyxcuFBdzvN36tmx7/Dhw14ds4rid9+kSRMGDRpEt27dePrpp3Mtf+Zra3OSn2tuR40aRbdu3XjooYcK9AUmLi5O7Zfh+ZNdK1FxIjXbW1iFChWYNGkSr7/+Ona7nZdeeinLMp5NreCqrbr/6J1OpzrKS/PmzTEYDGqttnTp0gVq7jxz5gwffPBBjr0shw0bVuiRcZ599ll+++03Tp48yc8//0ynTp285teoUUMNfrjeiQngs88+47PPPgPyrq0WROPGjRk1ahQff/wx8fHx2X6I5dZjsyB69+7N2rVr2bBhA+fOnctyGVVAQABvvvlmkewrv3r27MmCBQtwOp18/fXXfP3114Drd3GjGjZsyDPPPMPs2bNxOBy88cYbXq02EydOpGPHjvTs2ZO+ffvy/fffc+7cOYYMGZJlW+5zhzfTlClTePLJJ0lJSWHu3LnMnTvXa37Pnj3Vpl1w/e25z4GOGTMGcP3dbtmypch+96+++mq+yu5Zi508eTJPPPGE1/zt27ernbNWrVrFmDFjcmw5aNWqFa1atcrXfj0dPXpU3YengrY63GwStre4xx57jDp16rBw4UL27t3LtWvXCA4Opnz58rRo0cLr2z9A69atvULVHQhms5nGjRuzd+9ewDXMm7uXcl4aNGjAyy+/TFRUFEePHuXatWskJiYSHBxMnTp1GDBgwA196IWGhvL1118zb948Nm3axJkzZ1AUhTvuuIPu3bszcuRISpUqpS7fqFEjXn31VRYuXMi5c+eyPZ9YFJ5//nlatGjBkiVL2L9/P1arFYvFQqVKlWjdurXXB+qN0Gg0zJgxg+XLl7Nq1SqOHTtGWloa5cqV46677mLUqFE+HUYyO82bN2fGjBl8+OGHnDx5krCwMB599FHCwsKYNGnSDW//ueeeo0WLFixevJi//vqL+Ph4goODqV27tjr8pkajITIykk6dOvHNN99w+PBhkpKSCAkJoXLlyrRr1y7bpmVfa9GiBatWreKTTz7ht99+48qVKxgMBu6880769+/Po48+6hVQgwYN4syZM2zatImYmBivjoQ383d/4cIFtXOkwWCgX79+WZbp0KGDegmh+5rbovpSWdJplFt92A4hhBDCz+ScrRBCCOFjErZCCCGEj0nYCiGEED4mYSuEEEL4mIStEEII4WMStkIIIYSPSdgKIYQQPiaDWhSSoig4nf67RFmr1fh1/0KAvA+F//nzPajVavI9traEbSE5nQqxsXmPEeoLer2WkJAgrNZk0tOz3pZOiJtB3ofC3/z9HgwNDUKny1/YSjOyEEII4WMStkIIIYSPSdgKIYQQPiZhK4QQQviYhK0QQgjhY9IbWQhxW3JdvufE6XT4uyiikJxODampOmy2NByOor/8R6fT5/u+3XmRsBVC3FYURSElJZHExHgJ2lvAlStanE7fXfYTEBCMxRKa7+tpcyJhK4S4rVitsaSkJGI2B2E2B6LV6m74g1T4j06n8UmtVlEUbLY0EhOvAVC6dNkb2p6EbQmVZEsmNT0VnWKQDwoh8snpdJCSkkRwcBmCg0v7uziiCOj1Wp8NaGE0mgBITLxGqVIhN9SkLGFbAimKwum488TEWTFoDFgMwQQYAjDpTBh1Bn8XT4hiy+FwAAomk9nfRRElhDtwHY50tFpjobcjYVtCORQnGg3YHDYu2C8BYNQZCdAFUMoYhFlvxqwzodPq/FxSIYojaQ0S+VNULYcStiWYXqvHmPFNS1EUbE4bifZE4mzx6NBi0BkINgQRZAjErDdj0hnRauRqLyGEuNnkk/cWodFoMOlMWIylCDWVIdgQhAYN11LjOJVwlui4E0THneBi0iXi0xKwOWz+LrIQ4gbNn/8xHTq0Un/69u3Oc889w59/7rvhbV+4cJ758z/mypWYQq0/duy/6dChFa++OjHLPLvdTu/eXenQoRVLlnyZ7fpPPvk4HTq0Yu/e3dnOdx/zypVfZ5l38OABdf7hwwe9yvTii+MLdTw3SsL2FqXT6gjQmyltshBqKkOgPoB0p4NLyTGcsJ7ieNxJTsSf4krKVRJtSaQ70/1dZCFEIZhMJubO/Zy5cz/nhRcmEB8fz3PPPUN09PEb2u6FC+f5/PNPCx22AAEBgfzyy88kJyd7Tf/tt19IT8/5M+fUqZMcPXoEgI0b1+W6/Q0b1maZvnHjegICAgtZat+QsL1N6LV6ggyBhJjKEGIsjUGrJ9mewrmEC/wTf5LjcSc4k3CO2NRrJNtTcCpyyzQh8sPhcLBr107Wrv2eXbt2ZnTCunm0Wi2NGjWmUaPG3HNPd6ZNm47D4eDbb7/Jsqzrcpab16rVuHFTzGYzP//8k9f0jRvX0alT5xzX27BhLTqdjpYt27B162bsdnu2y3Xs2Jk///yDS5cuqtOcTidbtmzMdfv+IGF7G3I1ORspZQwmxFwGi7EUWjTEpcZz2nqO6PgTHI87wYWkS8SnWUlNT0NR5AbhQmS2efMG+vTtzsiRQ5k4MYKRI4fSp293Nm/e4LcyVaxYkdKly3Dhwnneeus1hgx5hN9+28HQoY9xzz3t2LFjOwDbt//E8OGD6Nr1bu6/vxfvvfeOWgPdu3c3zz77NABPPfWE2iTrdvHiRV5++SXuvbcL3bq159lnn/ZqrnXT63V06dKdTZvWq9OSk5P49def6d793hyPYePGdbRo0YqBAweRmJjAb7/9ku1ytWvXoUaNml6v9549u0hIsNK5c7cCvGq+J2Er0Gq0mN1NzuYyBOkDcTodXE6O4UT8aaLjT/BP/CkuJ18hwZaIXZqchWDz5g1ERDyHLVShybMduGvqfTR5tgO2UIWIiOf8FrhJSYkkJFgJCysHwJUrV5gx4z0GDnyc996bxZ131mHHjm1Mnvwf7rijOm+99T+GDh3B+vU/MnFiBAB169bj+edfAmDSpFfVZmpwheW4cf/m8OGDPP/8S7z22lvY7TbGjRvFqVMns5SnR497iYr6nbi4OAC2bdtKQEAgrVu3zbb8Bw7s5/z5c3Tv3ovWre+iTJky2TYVe27fs6l548Z1tG17N8HBwQV+7XxJeiOLLPRaPXqtnkACURQFuzOdtPQ0EmyJaDQajDoDgboAgo3BBOhNmOQSI3ELcPXoz765MjOHw8G7kW8T0qAC9Ya3RqN1XR5SqkYo9Ya35vDnu/hf5FTadeyITpf334ZRe2OD07jPf8bEXObDD9/H4XDQpUs3Nm1aT0KClffem0mDBo3U5V99dSL16jXgjTemqtMsFguvv/4ye/fupkWLVtSoUROAWrXCqVevgbrcDz+s4eLFCyxcuIxatcIBaNmyDf/6Vz8WLVrA5MmveZWtSZOmlCtXnq1bN9G//7/YuHEd99zTHb0++/jZuHEtRqORzp27otfrueeeHvzww3ckJSUSFJQ1QHv1updPPpnNyZMnqFy5Ctu3b+XFF18u3AvpQxK2IlfucHUPluFUnNgddhLsiVxLi0On0WHUGV2XGBkDMetc4SujWomSRFEUpu+dzT/xp/K1fPzxK1y6cJEmD3dQg9ZNo9VQpVtt9s/cwb8XjKF07bA8t1erdA2eb/FMof5uUlJS6NLlLvV5qVIW/u//XqRt23Zs2rSeMmXKeAVtcnIyx44dZfTo57y2c8893Zky5VX++usPWrRoRU7+/HMfNWvWUoMWIDAwkPbtO2bbC1qj0dC9ey82blxHly5d2bNnF8OHj8x22w6Hgy1bNtGuXXu1Ztqz572sWrWCbdu2ct99/bKsU6VKVRo2bMzGjeu48846KIpC+/Yd+fvv/Tkegz9I2IoC0Wq0mPQmTGSMquJ0YHPauJoaS0zKFQw6AyadiVKGYAIMZsw6s4xqJUqI/AedzZoKQGAlS7bzAytavJbzJZPJxEcffQpoKFOmDOXLV/AaVrBMmVCv5RMTE1AUhbJlvcf61ev1lC5dBqs1Ptf9JSQkEBqadZzg0NCyWK3WbNfp0eNeFi9eyJIlX1K+fAUaNWqS7XK7du3k2rVY2rfvREJCAgA1atSifPkKbNiwNtuwdW2/FytWLOPkyX/o1OkeTCZTrsfgDxK24obotDoCtAEE6AMAsDvs2Bx2LmYzqpVJbyJAZ5YmZ1HsaDQanm/xTL6bkfcE7WL0ohEkX7BSqkZolvnJF12hM77jGFq2ap3n9m6kGVmr1Xo182aWebPBwaXQaDTExl71mp6enk58fBwWS+5jRlssFk6fPpllemzsVSyW7L98hIfXpmbNWnz11WIef3xojse6caPr3Ozbb78OvO4178qVGK5evULZsllbCrp168msWe9z4cJ5IiNn5lp+f5GwFUXKoDNg0BkIyjjf6x7VKt4WjwYtRq9RrVxNzjKqlSgO3L3086Ntq7uoWKkSZzcf9zpnC6A4Fc5tPk7FypVp2+qufJ2zvZkCAwO58846bNmyiYEDB6vTt23bgsPhoEmTZgAYDK4WqbQ070uFmjRpxk8/bebEiX+oWbMW4GrK/vXXn7n77o457vfxx4fy00+buffePtnOT01NZfv2bXTs2IWHHx7oNS8uLo5XXpnA5s0beOSRQVnWDQkJZeDAwVy8eJ6WLfP+cuMPErbCZ9yjWpl0riYdp+IkzWHjWmocMSlXMWj16iVIAfpAAvQmDDfYUUSIm0Gn0/GfiIlERDzH4c93UaVbbQIrWki+aOXc5uNcO3iJyMgZxS5o3Z588t9MnBjBq69Oonfvvpw/f46PP/6Qli3bqOdr77ijOjqdjh9++BadToter6devQb06dOP5cuX8OKL/8fIkc8QGBjA4sVfkJaWxuDBw3LcZ69e99Gr1305zt+xYxspKck8/PDAbM8ZL13agA0b1mUbtgDPPDMuX8d+9epVtm7dlGV6u3YdMJt9d4MKCVtx02g1WgL0ZgL0rjd0ujOdNIeNS8kxKLia0sx6MxZjMGadGbPehF4rb1FRPHXr1pPIyBn8L3Iq+2fuUKdXrFyZyMgZdOvW04+ly12HDp2ZMuVdFiz4lIkTXyA4uBQ9e97nFVhlypTh//7vRZYs+YL163/E4XCwY8duAgODmDXrEz788H3ee28q6enpNGjQiFmzPqZ69RqFLtOGDeuoUKEizZu3zHb+vff25f333+X06VNUq1a90Ps5cuQQ//3vhCzTV6z4jkqVKhd6u3nRKDJaQaE4HE5iY5P8sm+dTsN5+zkSElMxUvhbPhUnrkuM7KQ5bKQ70zN6QRsJ1AcQbAxSw1eanIsPvV5LSEgQ164l+ex+okXNbrdx9eoFypathMFQNH87DoeDvXt3c+VKDGFh5WjRolWxrdHeinx5P1vI/T0TGhqETpe/zySpNohiwR2uxoxzZk7Fic1hx5qWQGxqHDqtFpPORLAhiEBDAGad6y5G0uQs/E2n0+U4QIMQbhK2olhyjWplwqx3ne9Nd6Zjc9i4knIVZ7KCQafHrDNnnO913bvXIJcYCSGKKQlbUSJcH9UKr1GtEu2JgOsSIxnVSghRXBWLsD1x4gRTpkxhz549BAQE0KdPHyIiIvLsGTZkyBCioqKyTP/xxx8JD78+ukndunWzLBMWFsYvv2Q/uLUo3jKPaqUoCjaHLddRrYw6o5zvFUL4jd/D1mq1MnToUCpXrszMmTOJjY1l6tSpxMXFERkZmef6LVq04KWXXvKaVrVq1SzLDRkyhL59+6rP3deQiZJPo9HkOqqVXqvHrDd7ne+VUa2EEDeT38N22bJlWK1WVq9eTWioayQWnU5HREQEzzzzjFcNNTsWi4VmzZrluZ9KlSrlazlR8mUZ1SrjfO8l+2XAPaqVmWBjMGYZ1eo2JRdhiPwpqgt2/N6utn37dtq1a6cGLUCvXr0wGo1s27bNjyUTtwqDVk+QIZAQUxnKGEuj1+hJsidzNuE8/8Sd4ljcP5xNOM+11DhS0lNwKiXjMhZRcK5LcjSkpfl+zGJxa7DZ0gDQ6W6sbur3mm10dDQPPfSQ1zSj0Ui1atWIjo7Oc/2oqCiaNWuGw+GgadOmPPfcc7RunXW4rk8++YTp06cTEBBAhw4dePHFF6lc2XcXMIviyT0kn8njEiP3qFZXUq6i9xrVKgCz3nzDtz8TxYdWqyMgIIjExDjS0+2YzYFotTr5/ZZgTqcGh6PoWyoURcFmSyMx8RoBAcFeN3coDL+HrdVqzXbwaovFQnx87nefaN26NQ888AA1atTg8uXLzJ8/n+HDh/Pll1/SvHlzdbkHH3yQLl26EBYWxtGjR5kzZw6DBg3i22+/pXTp3Afdzo1e75+GAa1WA3bQajT5vqBaZE+HFoNBDwQC10e1upJ2FSVVwagzYNaZKGUsRaAhAJPehEFGtQJQ33sl7T0YGhpGcrKZ+PhrpKb6Z2AaUVQ0aLUanE4FX50aCAoqRZkyZW/4C1mx/dRQFCXPg3v22We9nnfp0oW+ffsye/ZsPv30U3X6O++8oz5u3bo1LVu2ZMCAASxfvpyRI7O/r2JetFoNISFBhVr3RimKAqkQFGQk0OC7sTxvd65eznbS0tOId8ZiTddiUowEGwOxmEoRYHANPXmj33hLOoslwN9FKLDQ0GCqVKmAw+FQb7wuRGYGg6HIRgPze9haLJZs74GYkJCQZ+eozAIDA+ncuTPr16/Pdbl69epRs2ZN/v777wJt35PTqWC1Jhd6/RuhzbjDSFKSjXSdNH/5ngEjBpyKk+RUG9esiTicF9BrdBj0RizGYAIzmpxvp1GtdDotFksAVmsKDoec5xY3n6/fgykpjlznWywBJWe4xvDw8CznZm02G6dPn85yLjc/8ttzrCh6mPlrPFhdRsA6FQVHCRmT9lZhwIhBZwSd6xKjNLuNi2kxOBXXqFYmnQmLIZgAQ8BtM6qVw+EsMWMji1tTSXgP+r39q1OnTvz+++9cu3ZNnbZx40ZsNhudO3cu0LaSk5PZtm0bjRs3znW5Q4cOcfLkyTyXEyI3Oq2OQEMAZUylCTGVxqQ1YXPYuJB8iX/iT3I8/gSn4s9wNSWWJHsyDmfu35KFELcuv9dsBw4cyKJFixg9ejSjR4/m6tWrTJs2jX79+nk1I0+aNInVq1dz8OBBAHbv3s38+fPp0aMHlStX5vLly3z++efExMQwY8YMdb358+dz5swZ2rRpQ2hoKMeOHWPu3LlUrFiRhx9++KYfr7g15TqqlS0eHVp1VKtAw/UmZxnVSoiCcypOHIoTR7pCil1XZNfC+pLfw9ZisbBw4UKmTJnCuHHjMJvN9O3bl4iICK/lnE4nDsf1mkG5cuWw2WxMnz6duLg4AgICaN68Oa+//jpNmjRRl6tZsyYbNmzgxx9/JCkpiZCQEDp37sz48eOz7QUtRFHIdVSrVAW9RpdpVCuTescjIW437vB0Kk6cisMVpE7XY6fiJN3pIN2Zjt2ZTrqSjsPpwIkTjQZCnMGU1YahL+a3G5X72RaS3M9W3Aj3qFY2pw0FMGqNBOhdlxiZ9SbMOhP6Yn6JUUm8n624OQobnk7FidPpxKlcv5RHATS47gTm/eO69NFg1lDRWBmDHz4L5X62QhRzBq3eNbIVga4mZ6edZHsK8WkJaDVajDoDQYZAgg1BmDPuYiRNzsJfbkZ46jU6tFojWp0GrUabv179WgUoGX0hJGyF8LPsRrWyOWzEpcZzNSUWvVaPMWNUq0AZ1UoUgbzC0+F0YPdHeN7CJGyFKGa0Gi1mvRmz3jVgSXpGk/Pl5BhQwKAzqOd7AzKWk1Gtbm8SnsWf/IUKUczptXr0Wj2BGU3OdqedVHsq1rQEtRd0kD6QYGMQZp0Js94sTc4lXFGFp7tDjoSn/0nYClGCuMLVqPZcdjU527GmJRCbeg2dRofJfYmRMRCzznW+Vz5I/euGwlNxqmP/SniWXBK2QpRgriZnE2b99UuM0hw2rqTG4ky5gkFnuC1HtfI1dwg6iiA83ZGYY3gi4XkrkLAV4hai0+oI1AYQiOvmAHaHnTSHjQv2SwAYdUYCdAGUMga5zgvrTOi0RTPQekkm4Sl8TcJWiFuYQWdQa7KuS4xsJNoTifMY1SrIEEiQIfCWGtUqu/B0h2aRhqfGoD6X8BS5kbAV4jbhusTIdQ4X3KNa2bmWGseV1Fj0Gh0mnYlSxiAC9IEE6IvPqFYFDU+n04FDwlMUIxK2QtymdFodAVodARmXGLlHtbqUHJN1VCud67xwUY1qJeEpbjcStkIIIOdRra5fYmT0GtUqSHf9pvH5Cc90Zzo2CU9xm5KwFUJkkdeoVjqtjkBjAHGKhdj4BGz2dK/wVBTF9YOEpxAgYSuEyIfsRrVKV9KxpiWS5rADGglPIXIhYSuEKDC9Vo9Jb8RiNqOxp+KQu/4IkauS38dfCCGEKOYkbIUQQggfk7AVQgghfEzCVgghhPAxCVshhBDCxyRshRBCCB+TsBVCCCF8TMJWCCGE8DEJWyGEEMLHJGyFEEIIH5OwFUIIIXxMwlYIIYTwMQlbIYQQwsckbIUQQggfk7AVQgghfEzCVgghhPCxYhG2J06cYMSIETRr1ox27doxZcoUUlNT81xvyJAh1K1bN8tPdHS013J2u5333nuPDh060LRpU4YMGcLhw4d9dThCCCGEF72/C2C1Whk6dCiVK1dm5syZxMbGMnXqVOLi4oiMjMxz/RYtWvDSSy95TatatarX86lTp7J69WomTJhAlSpVmDdvHsOGDWPNmjWUK1euSI9HCCGEyKzAYXvt2jVCQkKKrADLli3DarWyevVqQkNDAdDpdERERPDMM88QHh6e6/oWi4VmzZrlOP/SpUssW7aMyZMn88gjjwDQtGlTunXrxsKFC4mIiCiyYxFCCCGyU+Bm5M6dOxMREcHu3buLpADbt2+nXbt2atAC9OrVC6PRyLZt2254+zt27MDhcNCnTx91WnBwMF27di2S7QshhBB5KXDY2mw2fvjhB4YMGULfvn1ZtGgRiYmJhS5AdHR0ltqr0WikWrVqWc69ZicqKopmzZrRuHFjBg8ezK5du7JsPywsjDJlynhNDw8P58SJEzidzkKXXQghhMiPAjcj33nnnRw7dgxwBdlbb73Fe++9R58+fXj00Udp3LhxgbZntVqxWCxZplssFuLj43Ndt3Xr1jzwwAPUqFGDy5cvM3/+fIYPH86XX35J8+bN1e2XKlUqy7qlS5fGbreTnJxMcHBwgcrsptf7p3+ZVqsBO2g1GnS6YtHHTdyGtFrN9X/99Lcgbm8KThRAp9Wi1xbv92CBw3bNmjUcPXqU7777jh9++IELFy6QkpLCN998wzfffEP9+vV54oknuP/++9HewMErioJGo8l1mWeffdbreZcuXejbty+zZ8/m008/Vadntx1FUQpdNnB9wISEBN3QNgpLURRIhaAgI4EGs1/KIIRbcJDJ30UQt6l0p4PEtHRKWcwEGgL8XZxcFao3cp06dYiIiCAiIoKoqCg+/PBDoqKiADh06BATJ05k3rx5fPzxx1SpUiXXbVksFqxWa5bpCQkJeXaOyiwwMJDOnTuzfv36PLdvtVoxGAwEBgYWaB9uTqeC1ZpcqHVvlLtGkZRkI12X+xcSIXxFq9UQHGQiMSkNp/PGvrwKURgKTtBDgjWVNO3NPyVosQTku3Xxhi792bp1K4sWLWL37t1oNBoURVFrjNHR0bz55pvMnTs3122Eh4dnOTdrs9k4ffo0Dz30UIHLlLnGGh4eztWrV4mLi/M6bxsdHU3NmjVvqPadnu6f8726jIB1KgoOP5VBCHfTsdMp70PhJ1oFDeBwOkkv5v1vCpw0VquV+fPn0717d0aPHs2vv/6K0+lEo9HQs2dPvvrqK2bNmgWQrx7LnTp14vfff+fatWvqtI0bN2Kz2ejcuXOBypacnMy2bdu8zht36NABrVbL2rVr1WlJSUls2bKlwNsXQgghCqPANdtOnTqRlpYGuGqRZrOZ/v37M3z4cKpVq6YuV7VqVc6ePZvn9gYOHMiiRYsYPXo0o0eP5urVq0ybNo1+/fp5NSNPmjSJ1atXc/DgQcAV5PPnz6dHjx5UrlyZy5cv8/nnnxMTE8OMGTPU9SpUqMDAgQOJjIxEr9dTuXJlPvvsMwCGDh1a0MMXQgghCqzAYeseRjE0NJRBgwbx+OOPZzvIxb333suVK1fy3J7FYmHhwoVMmTKFcePGYTab6du3b5bBJpxOJw6HQ31erlw5bDYb06dPJy4ujoCAAJo3b87rr79OkyZNvNadMGECgYGBfPDBByQkJNC0aVMWLlwoo0cJIYS4KTRKAbvl9urVi2HDhjFgwABMptu3F6LD4SQ2Nskv+9bpNJy3nyMhMRUjRr+UQQidXoullBlrQqqcsxX+oVXQGB1UNFbG4IfPwtDQIN91kFq3bl2el+QIIYQQ4roCd5BatmwZY8eOZcWKFV7Tly9fztixY1m6dGmRFU4IIYS4FRQ4bL/66is2b95M7dq1vabXq1ePTZs2sXz58iIrnBBCCHErKHDYunsY16tXz2v6nXfeCcCZM2eKoFhCCCHEraPAYeu+7CcuLs5ruvu5zWa74UIJIYQQt5ICh2358uUBmD17ttdoTbNnz/aaL4QQQgiXAvdGbtu2LStXruTrr78mKipKHW7x9OnTaDQa2rZt64tyCiGEECVWgWu2I0aMwGh0Xc90+vRptm7dyunTp1EUBaPRyIgRI4q8kEIIIURJVuCwDQ8PZ9asWYSGhqo3HlAUhbJlyzJz5kxq1arli3IKIYQQJVah7vrTuXNntm7dyp49e7hy5QphYWG0bNlSrfEKIYQQ4rpC32LPaDTSrl27oiyLEEIIcUsqVNharVbWrFnD8ePH1RsTuGk0Gt5+++0iKZwQQghxKyhw2J47d47HHnuMmJiYLPMURZGwFUIIITIpcNjOnj2by5cv+6IsQgghxC2pwL2Rd+7ciUajYcCAAYCr2fjll1+mWrVq1KxZU2q1QgghRCYFDlt3rdbz5u6DBw9m5syZnDhxIl83jBdCCCFuJwUOW/e9bMuUKYNe72qFtlqt1KhRA3DdFUgIIYQQ1xX4nK3FYuHKlSskJSUREhLClStXmDJlCiaTCUBqtkIIIUQmBa7ZVqtWDYCLFy/SqFEjFEVhzZo1fP3112g0GrWGK4QQQgiXAodthw4dqFu3LqdPn2bEiBHodDp1yEaNRsPYsWN9UU4hhBCixNIonvfJK4S//vqLH374AZ1OR8+ePWnWrFkRFa14czicxMYm+WXfOp2G8/ZzJCSmYkSGyBT+odNrsZQyY01IxZHu9HdxxO1Iq6AxOqhorIzBD5+FoaFB6HT5q7MW6Jxtamoq8+bNQ6PR8OCDD1KlShWaNGlCkyZNClVQIYQQ4nZQoLA1m83MnTsXh8PBkCFDfFUmIYQQ4pZS4HO27g5Q6enpRV0WIYQQ4pZU4LB98sknURSFTz/91BflEUIIIW45Bb7OdteuXYSEhLBgwQK2bNlCgwYNMJvN6ny5EYFvORwO9u7dw9GLRzAHlaJpk+bodDp/F0sIIUQuCtwbuV69euooUjk5dOjQDRWqJPBHb+TNmzfwv8ipXLxwQZ0WVrE8T415hrs7d7ypZRFCeiMLf3I4HBzcv59r8THUrliH1i3uuukVD5/1RnbLLZ/zCmJROJs3byAi4jlCGlSgycMdCKxkIfmClbObj/POK2/w0huvSOAKIW4Lv277mXkfzeHKxet3oKtYqRL/iZhIt249/ViynBW4ZhsVFZXnMm3atCl0gUqKm1mzdTgc9OnbHVuoQr3hrdFor3+hUZwKhz/fhfaqk0+WfKGOVy2Er0nNVvjDr9t+5p1X3iCkQQWqdqvtVfG4dvASkZEzblrgFqRme8ODWtyubmbY7tq1k5Ejh9Lk2Q6UqhGaZb71ZCz7Z+6g8Zj2VKhbBbPOhEn9Mbr+1RvV5+bM8zz+1Wnl/K/IHwlb4UtOxUmaw4bNYVP/TbGl8ua/X0Jb3pRjxcN4TcsPazbelCZlnzcjF7UTJ04wZcoU9uzZQ0BAAH369CEiIsKr41VeNm7cyNixY7nzzjv5/vvvvebVrVs3y/JhYWH88ssvN1z2m+HKlRgAAitZsp0fWNE1PS0+hZT0VFLSUwu9L4NWnymArz82692Pved5hrdRZ5RTCULcxhxOBzanzSsoM4dmmtP9PC3beWkOG+nOrJeXxh+/QlzMNZo81sEraAE0Wg1VutVm/8wd7N27m9at296sQ86XAodt/fr1c52v0Wg4ePBgvrdntVoZOnQolStXZubMmcTGxjJ16lTi4uKIjIzM1zZSU1OZOnUqYWFhOS4zZMgQ+vbtqz43GAz5LqO/hYWVAyD5gjXbmm3yRSsA/2rWn/CGdUhLT1PfxGmONFI9Hqc5bKSlZ51md9oBsDvTsTvTSbQXvtaefRi7a9jZ16g9A1uv1UtgC3GTOZwOV+B5BWXOYWjLLkQdaaQrjiItl16rx6R1fcm3pyUAeVc83BWU4qTAYVvUrc7Lli3DarWyevVqQkNdQaLT6YiIiOCZZ54hPDw8z218/PHHVK5cmapVq3LgwIFsl6lUqVKJHbe5RYtWVKxUibObj2fbdHJu83HKVSxP65ZtCt104m6ycf1xpZGWbssUyK7HqR7z0zLNdyiupkT3N1lIKFRZtBrt9dp0drVsj8A2S3O4uM2lO9OvB54zh5qkZ3g6s85Lc9hwFHFIGrR6jO6/Sa0x43HWf006I0at57TrrWRGrcHrb3l/+h9EsTXPioe7glKcFDhsK1eu7PXc4XBw5coVHA4HBoOB8uXLF2h727dvp127dmrQAvTq1YtJkyaxbdu2PMP29OnTfP755yxbtowFCxYUaN8lhU6n4z8RE4mIeI7Dn++iSrfaBFa0kHzRyrmMTgEvvfHKDZ2j0Gq0BOjNBOjz33SfWboz/Xogp2cf1LZcA9uGgoJTcfqgOTxTbVufQ1O5NIeLm0BRFByKI1PgpWVqYs2hGdbpvbz7S25RMWgN3qGozRyOJu/A1HqEpse/Wk2Bx0zKU4MmjQmrWD7XikfFypVp0aJVke/7RhU4bLds2ZJlWnJyMtOnT2fZsmW8++67BdpedHQ0Dz30kNc0o9FItWrViI6OznP9t956iwceeIB69erlutwnn3zC9OnTCQgIoEOHDrz44otZvjgUZ9269SQycgb/i5zK/pk71OnlKpYvNpf96LV69Fo9QYbAQq2vKAo2p92rudszsK/XtDOmpXvXtm0OGzafNIfn0LlMn31HM/Nt0BzucDj4+6+/SE2yYg6yUK9hw9ticBVFUUhXHFlri1nORXrPz67m6SzikDRqDdnXHrUm71pk5qDUX695GnUGn4RkUdHpdDw15hneeeWNHCsekZEziuV7sUg6SAUGBjJp0iS++eYbZs6cycKFC/O9rtVqxWLJ2v5usViIj4/Pdd0tW7awb98+1q1bl+tyDz74IF26dCEsLIyjR48yZ84cBg0axLfffkvp0qXzXdbM9Pqb+6bs1eteunfvwb59ezh8/hCBwRaaNW9RLN9YhaVHRyCFr12rzeHu89Kegex5rjrTY1eYux67m9OuN4cXjqs53IQ5UzCbs3us92gS93hcHJvDf/lpO5/OnE2MxzWO5SqWZ+Szo2nfpZMfS5YzRVHUlhd3CKZ6hKAajOlpZK5JpmVarshD0iMIMzelqs/1Jrxrl5maW4t5SBaljt06o9W9yqczZ3tVPCpVrsz06TPp0aOXH0uXsyLrjXzhwgVsNht//fVXkWzPfTP6nKSlpfH2228zbtw4rybo7Lzzzjvq49atW9OyZUsGDBjA8uXLGTlyZKHKp9VqCAkJKtS6N6p79y5UulwZUAg0BPilDMVb4WrWbnZHOqnpaRk/qerjFLv389T0NFI8n9uvT7/eHJ5CSnoKpBWuLAatHrPejNlgIkDv6hHu+jGrjwMMGT3F9SYCPKfrTRj1JrRFWLvetuknpk5+3TW4yiPeg6tMnfw6b05/i87duxTZ/hRFwe60k5qe8QUpPe36Y3ffgvSML1Lq9OvLqvMdaTiLsL+JBjC6fxfuL0j666Ho/h2Y9Eb1d2PWeTzOWNbV3Hprtn74Uu9+Pel5Xzf27d7HuQvnaFa7MZ3adyrWFY8Ch+0TTzzh9VxRFFJTUzl+/DgOh4NSpUoVaHsWiwWr1ZplekJCQq7naxcuXIhWq6VPnz7q+na7HafTidVqxWw2YzRmfzPhevXqUbNmTf7+++8CldWT06lgtSYXev0boc04T5GUZCNdJ3+ovqHDRCAmbSCljVCQ+1KrzeGZatVpjowAcPcS92wmd4dExmOv5nBbIgm2xEIfSeae4OZcatLqY49l3c3hDoeDGe98QEiDCl7ny0rVCKXe8NYc/nwXM96dQZNWrdFqtdid9uvN/unetcPMTbCZz096Lq9QtJ0yTdnVHLOtPXo3v3rON2oNhT9FoAB2sNkVbIX9FiYAuLNBA2o3qUMVc1Ws1sL38SgsiyXAd9fZRkVFZfsmc/dS7tq1a4G2Fx4enuXcrM1m4/Tp01nO5Xr6559/OHXqFO3atcsyr3Xr1rz22ms89thjOa5fFL2q0/10Ib8uI2CdiiKDCRRTevTodXqCdIVr/fDuHZ6pqTu7XuJqOF1vHs/SHG67sd7hidHXiLl4mSaP5H6N46tfvUlQrZAiDUkNmqznHbW5dNrx6NjjOc1wIyHppoDTobgeCP/SKmgAh9NJurN4fxYW2djIJpOJBx54gBdffLFA2+rUqRNz5szh2rVrhISEAK4BKmw2G507d85xvZEjR9K/f3+vaZ988gknTpxg6tSp6n13s3Po0CFOnjyZa5gL4U9F2TvcM5A9Azq7wM6td3js1atA3tc4JsUlEkgZwBWSnucW8wrDLOGZEaiGW7izmbg9FDhsN2/enGWayWTKdUCJ3AwcOJBFixYxevRoRo8ezdWrV5k2bRr9+vXzakaeNGkSq1evVgfMCA8Pz9LMvGrVKi5dukTbttdHDpk/fz5nzpyhTZs2hIaGcuzYMebOnUvFihV5+OGHC1VmIUqCougdbnfa1ZryftufHGVvntc49m96P02bNsekvbV7ZAtREAUO2ypVqhRpASwWCwsXLmTKlCmMGzcOs9lM3759iYiI8FrO6XTicBT8ouuaNWuyYcMGfvzxR5KSkggJCaFz586MHz8+217QQggXjUaT0dPVdcK6c9vOLKo4L8/BVe5q1a5Yd1QRwh8KfCOCDRs2sGfPHpo1a0bv3r3V6T/++CN//vknLVu2pGfP4nmLo6Lkj/vZuul0Gs7bz5GQmIqxID13hLhBnndcyWlwleJwzbe4TWgVNEYHFY2VMfjhs9CnNyKYN28e+/fvp0uXLl7Ty5Yty8KFC/nzzz9vi7AV4nZ0d+eOvPTGK8z7aE6xHVxFiOKowGF76tQpABo3buw1vWHDhoDrDj5CiFvX3Z070rbD3Rz+++/bbgQpIQqrwGGblORqOk1LSyM4OFidnpbmul4sOdk/154KIW4enU5HkxbN5H62QuRTgcf3Klu2LACLFi3ymr548WKv+UIIIYRwKXDNtmXLlvz444/MnTuXvXv3UqdOHY4dO8bOnTvRaDS0bNnSF+UUQgghSqwCh+3QoUNZt24diqIQFRVFVFQU4LomT6fTMXTo0CIvpBBCCFGSFbgZuWnTprz22msYjUYURVF/TCYTr732Gk2aNPFFOYUQQogSq1DDNT7yyCN06dKF7du3c+XKFcLCwujUqVOBbxwvhBBC3A4KfYu98uXL869//asoyyKEEELckgrcjLx06VLGjh3LihUrvKYvX76csWPHsnTp0iIrnBBCCHErKHDYfvXVV2zevJnatWt7Ta9Xrx6bNm1i+fLlRVY4IYQQ4lZQ4LA9e/Ys4ApXT3feeScAZ86cKYJiCSGEELeOAoete6SouLg4r+nu5zab7YYLJYQQQtxKChy27h7Hs2fP9rqJ/OzZs73mCyGEEMKlwL2R27Zty8qVK/n666+JiooiPDyc6OhoTp8+jUaj8bpxuxBCCCEKUbMdMWIERqPrvoGnT59m69atnD59GkVRMBqNjBgxosgLKYQQQpRkBQ7b8PBwZs2aRWhoqNcIUmXLlmXWrFnUqlXLF+UUQgghSiyN4nnitQBsNht79uxRR5Bq2bIlf/31FytXruTtt98u6nIWOw6Hk9jYJL/sW6fTcN5+joTEVIwY/VIGIXR6rdxiT/iXVkFjdFDRWBmDHz4LQ0OD0OnyV2ctdNi6nT9/nlWrVrF69Wr1sqBDhw7dyCZLBAlbcbuTsBV+V4LCtlDDNaamprJu3TpWrVrFrl271KZkAI1GU5hNCiGEELesAoXt7t27WblyJevXryc5ORnAK2TLly/Pgw8+WOSFFEIIIUqyfIXtRx995NVM7NnyrNfrSU9PB+Cnn36Smq0QQgiRSb7CdtasWWg0GjVkzWYznTp14r777qNChQo89thjgDQhCyGEENkpUDOyVqtl4MCBREREEBgYCMDhw4d9UjAhhBDiVlGgsFUUhaVLl7J582Z69+5N79691QEuhBBCCJG9fIXtgAEDWL9+PUlJrktdLl26xMKFC1m4cCEWi8WnBRRCCCFKunxdIPT222/zyy+/MG3aNO666y60Wq16uY/ValXP1Q4YMIAlS5b4tMBCCCFESVOoQS0uXLjAypUr+e677zh16pRrQxkdqDQajQxq4WMyqIUoDmRQC+F3JWhQiwKPjQxQqVIlxowZw/r161m0aBEDBgxQO0wJIYQQwluhRpDy1KpVK1q1asUrr7zCunXr+Pbbb4uiXEIIIcQto1A12+yYzWYefPBBPv/88wKve+LECUaMGEGzZs1o164dU6ZMITU1tUDb2LhxI3Xr1qVv375Z5tntdt577z06dOhA06ZNGTJkiFyyJIQQ4qYpsrAtLKvVytChQ0lKSmLmzJm89NJLrFmzhpdffjnf20hNTWXq1KmEhYVlO3/q1KksXryYZ599ltmzZ6PX6xk2bBgxMTFFdRhCCCFEjm64GflGLVu2DKvVyurVqwkNDQVAp9MRERHBM888Q3h4eJ7b+Pjjj6lcuTJVq1blwIEDXvMuXbrEsmXLmDx5Mo888ggATZs2pVu3bixcuJCIiIiiPyghhBDCg99rttu3b6ddu3Zq0AL06tULo9HItm3b8lz/9OnTfP755znWhHfs2IHD4aBPnz7qtODgYLp27Zqv7QshhBA3yu9hGx0dnaX2ajQaqVatGtHR0Xmu/9Zbb/HAAw9Qr169HLcfFhZGmTJlvKaHh4dz4sQJnE65ZEEIIYRv+b0Z2Wq1ZjsKlcViIT4+Ptd1t2zZwr59+1i3bl2u2y9VqlSW6aVLl8Zut5OcnExwcHDBCw7o9f75rqLVasAOWo0m39d4CVHUtFrN9X/99Lcgbm8KThRAp9Wi1xbv96DfwzYn7gEycpKWlsbbb7/NuHHjvJqgs5PddgoxlocXrVZDSEjQDW2jsBRFgVQICjISaDD7pQxCuAUHmfxdBHGbSnc6SExLp5TFTKAhwN/FyZXfw9ZisWC1WrNMT0hIyLVz1MKFC9FqtfTp00dd326343Q6sVqtmM1mjEZjjtu3Wq0YDIZCD8bhdCpYrcmFWvdGuWsUSUk20nVyW0PhH1qthuAgE4lJaTidN/blVYjCUHCCHhKsqaRpb/4pQYslIN+ti34P2/Dw8CznZm02G6dPn+ahhx7Kcb1//vmHU6dO0a5duyzzWrduzWuvvcZjjz1GeHg4V69eJS4uzuu8bXR0NDVr1kR7A00P6X4aok6XEbBORZFh8oT/ZDQdO53yPhR+olXQAA6nk/Ri3v/G72HbqVMn5syZw7Vr1wgJCQFcA1TYbDY6d+6c43ojR46kf//+XtM++eQTTpw4wdSpU6lRowYAHTp0QKvVsnbtWvUm90lJSWzZsoWHH37YNwclhBBCePB72A4cOJBFixYxevRoRo8ezdWrV5k2bRr9+vXzakaeNGkSq1ev5uDBg4CrRpy5mXnVqlVcunSJtm3bqtMqVKjAwIEDiYyMRK/XU7lyZT777DMAhg4dehOOUAghxO3O72FrsVhYuHAhU6ZMYdy4cZjNZvr27ZtlsAmn04nD4SjUPiZMmEBgYCAffPABCQkJNG3alIULF1KuXLmiOAQhhBAiV4W6xZ6QW+wJIbfYE353q99iTwghhBD5J2ErhBBC+JiErRBCCOFjErZCCCGEj0nYCiGEED4mYSuEEEL4mIStEEII4WMStkIIIYSPSdgKIYQQPiZhK4QQQviYhK0QQgjhYxK2QgghhI9J2AohhBA+JmErhBBC+JiErRBCCOFjErZCCCGEj0nYCiGEED4mYSuEEEL4mIStEEII4WMStkIIIUocRVFQUPxdjHzT+7sAQgghijfPYMv6GNz/df0/45niOcdz3vV1yZinKApo3PsCjUaDgoJGQZ2OAoom46miQaMBnU5LWGBpdZHiTMJWCCGKkcwhpIYTngHmDi+POFOu1/O818sh6DQZwQau/yiARpOxrYyHuJbRajSARl1WgwZNxgIatOqyGUugRYNG45qu0WjRoEGn0bqW1rh+NGjRarVoAC3a68ujUZcDV/C6p7nnu/ev12sJDQkmNcGBw1G8a7kStkKI21aW2hXXa2YoXvGEa3bGNPVx1nWzCzbFHSIe6aYoCppMged6rMkIGNRgg+uh4w4512z3/7RotdeDTaPRoEXrCrWMxxrICDctWo07IN3bzD5A3VtHQ8YWNNkEYsazTOXLXFZf0Ou1BBoCSNMkQTFvUpawFULcNDezOZKMJkgF8myO9Aw097+etSnwDDYNWu31sNFoXFGm1WhBo3E9RoNGq82osV0PIY3Xtsk0L9M0dd/ey7gmaTLCL/sQFsWPhK0QIl8URcGhOEh3OnAoDhSHE5suhaSUNNIdzkzn2a43R5Ip2ArXHOmuoWWEGhq0Wi0Zj643R2p0XjW1nJojtZlqX+4QvF5LU+tx4J7vGXgSbKKAJGyFEF4cTgfpigOHM931r+JQz+HpNDr0Wj0GnYFgUwDlSpfGqknF4VAK0Ryp1uWuN0dmCjYyPRaipJKwFeI25FScaqimO9NdgZrRMKtDi16nR6/VEagLxKwzYdAZXCGr1WPQGtBpdej1WkJKBWFKTyI93envQxKiWJOwFeIW5W729QxVZ8aZTw1a9FodOo2OIEMgJr0Jo9aIQecKVL1Gj16rlxqlEEVEwlaIEu56s6+DdCU9x2Zfi7EUJr0JvVaHQWtwhapWn3EOVAjhSxK2QpQA+W/2LZVjs68Qwn+KRdieOHGCKVOmsGfPHgICAujTpw8RERGYzeZc1/vf//7HTz/9xPnz59FoNNSsWZMnn3ySPn36eC1Xt27dLOuGhYXxyy+/FOlxCHEjsm/2dZ0L1aBFp9Wi1+izNPvqNXq1lirNvkIUT34PW6vVytChQ6lcuTIzZ84kNjaWqVOnEhcXR2RkZK7rpqSkMHDgQGrWrImiKKxfv57nn38ep9NJv379vJYdMmQIffv2VZ8bDAafHI8Qecmu2dd9TahOo0On1Xk0+xozaqjXa6rS7CtEyeP3sF22bBlWq5XVq1cTGhoKgE6nIyIigmeeeYbw8PAc133llVe8nnfs2JHjx4+zatWqLGFbqVIlmjVrVuTlFyI7ns2+7lB1D8ig1biafXUaHRZD1mZffcaPEOLW4fe/6O3bt9OuXTs1aAF69erFpEmT2LZtW65hm50yZcqQlJRU1MUUIovMzb4OpwMHDsB1bahOq0Ov0RNgMGPWmzFqjR7nUaXZV4jbid/DNjo6moceeshrmtFopFq1akRHR+e5vqIoOBwOkpOT2bJlC7/88gv/+9//siz3ySefMH36dAICAujQoQMvvvgilStXLrLjELcuR8aISel5NPuWMgZj1puk2VcIkYXfw9ZqtWKxWLJMt1gsxMfH57n+b7/9xvDhwwHQ6/X897//5d577/Va5sEHH6RLly6EhYVx9OhR5syZw6BBg/j2228pXbp0ocuu1/vnQ1Sr1YDddScOnU4+yIuCoiiku0dMcmZ0TlJcnZN0Gq0rOA06SukCMevNGHQGrxrq7djs637vyXtQ+EtJeg8W208I1x0x8m5ia9KkCV9//TWJiYls376dN998E51Ox8MPP6wu884776iPW7duTcuWLRkwYADLly9n5MiRhSqfVqshJCSoUOveKEVRIBWCgowEGnLvsS2uU8f2daRjd7qHI8zo7avRoNfqCdCaMeoMBBoCMOmNrkDVGdRwlWbfrCyWAH8XQdzmSsJ70O9ha7FYsFqtWaYnJCTk63xtcHAwjRs3BqBdu3bYbDamTZvGgAED0Omyv7awXr161KxZk7///rvQ5XY6FazW5EKvfyNcdxyBpCQb6Tr58M/Mq9nXmY5TceDM+PLmGuRBh0Grx6QzE6w3q7VTd7BqNVpwAjZXa7ENBRs21wSh0um0WCwBWK0pOBwyXKO4+fz9HrRYAvJdq/Z72IaHh2c5N2uz2Th9+nSWc7n50bBhQxYtWkRsbCzlypXLcTn3bbluhL/Gg9VlBKxTUXDcpmPSupt9Pc+lun+nWo0WnVaPXqMjSB+ESWfCmNHb17PHb3acDtRrW0X+OBxOGRtZ+FVJeA/6PWw7derEnDlzuHbtGiEhIQBs3LgRm81G586dC7y9PXv2EBwcrG4rO4cOHeLkyZOFCnNx8+S3t6/ZYCZAb8aoNaDXGqS3rxCi2PF72A4cOJBFixYxevRoRo8ezdWrV5k2bRr9+vXzakaeNGkSq1ev5uDBgwAcPnyYyMhI7r33XqpUqUJycjJbt27l66+/5oUXXkCvdx3a/PnzOXPmDG3atCE0NJRjx44xd+5cKlas6HVeV/iPU3G6Oifl2ds3CLPe7FFDNaDPmC+EEMWZ38PWYrGwcOFCpkyZwrhx4zCbzfTt25eIiAiv5ZxOJw6HQ30eFhaGxWJh9uzZxMTEUKpUKWrVqsVHH31E9+7d1eVq1qzJhg0b+PHHH0lKSiIkJITOnTszfvz4bHtBC9/Ib7OvxVCqQM2+QghREmiUojh5eRtyOJzExvpn8AydTsN5+zkSElMxYvRLGbKTn2ZfnUaHUWfM0uzrCladXJNaguj1WkJCgrh2Te5nK/zD3+/B0NCgktNBSpQ8ns2+jowB8wFp9hVCiBxI2IpsKYqSUTu9HqruQR48m33N+ozOSdLsK4QQOZJPxNuY2uybcR41p2Zfs8GMWWfCpDNKs68QQhSChO1twN3s6znQg8L1QR50Wh16rV6afYUQwkckbG8R+W32DTJIs68QQtxs8glbgiXbU0hyuoaMzNLsqzVi0psymn116LUGafYVQgg/kbAtgTQaDRZTMM4ADXqMGHQZ51A1rlqqNPsKIUTxImFbQlUrU4VSilzfKIQQJYG0KQohhBA+JmErhBBC+JiErRBCCOFjErZCCCGEj0nYCiGEED4mYSuEEEL4mIStEEII4WNyP9tCUhQFp9N/L51Op8XhkGtshX/J+1D4mz/fg1qtBo1Gk69lJWyFEEIIH5NmZCGEEMLHJGyFEEIIH5OwFUIIIXxMwlYIIYTwMQlbIYQQwsckbIUQQggfk7AVQgghfEzCVgghhPAxCVshhBDCxyRshRBCCB+TsBVCCCF8TMJWCCGE8DEJWyGEEMLHJGxLiJUrV1K3bt0sP5GRkf4umriFnTp1ildeeYUHHniABg0a0Ldv32yX27ZtGw8++CCNGzemR48eLF68+CaXVNyq8vMenDBhQrafj9u3b/dDibOn93cBRMHMmzePUqVKqc8rVKjgx9KIW92xY8fYtm0bTZs2xel0kt0dOfft28fo0aN54IEHmDBhAnv37mXKlCkYjUYefvhhP5Ra3Ery8x4EuOOOO7JUPsLDw29GEfNFwraEadiwIaGhof4uhrhNdO3ale7duwOu2sOBAweyLPPRRx/RoEED3n77bQDuuusuLly4wIwZM3jooYfQaqUBTRReft6DAGazmWbNmt3EkhWM/BUIIXKUV1DabDZ+//13+vTp4zW9X79+xMTEcPDgQV8WT9wGbpUva7fGUdxG+vbtS/369enWrRsff/wxDofD30USt7HTp09jt9upVauW1/TatWsDEB0d7Y9iidvQ6dOnadWqFY0aNWLAgAFs2rTJ30XyIs3IJUS5cuUYN24cTZs2RaPRsGXLFj744AMuXbrEK6+84u/iidtUfHw8ABaLxWu6+7l7vhC+VL9+fRo3bkzt2rVJSEhg6dKljBkzhhkzZnDvvff6u3iAhG2J0bFjRzp27Kg+79ChAyaTiYULF/L0009Tvnx5P5ZO3O40Gk2BpgtRlIYOHer1vGvXrgwcOJCZM2cWm7CVZuQSrHfv3jgcDg4dOuTvoojbVOnSpYGsNVir1QpkrfEKcTNotVp69uxJdHQ0qamp/i4OIGErhLgB1apVw2Aw8M8//3hNP378OFC8Lr0Qt5ecLhHyFwnbEuzHH39Ep9PRoEEDfxdF3KaMRiN33XUXa9eu9Zr+/fffU65cOXlvCr9wOp2sX7+eO++8E7PZ7O/iAHLOtsQYMWIEd911F3Xq1AFg8+bNLF++nCeeeIJy5cr5uXTiVpWSksK2bdsAOHfuHImJiaxbtw6ANm3aEBoaypgxYxg8eDAvv/wy/fr1Y+/evaxYsYI33njjlrlsQ/hPXu/BlJQUJkyYQN++falWrRrx8fEsXbqUAwcOMGvWLH8W3YtGKW51bZGtKVOm8PPPP3Px4kWcTic1atTg4YcfZsiQIdIJRfjM2bNn6datW7bzvvjiC9q2bQu4hmucPn060dHRVKxYkeHDh/P444/fzKKKW1Re78G6desyceJE/v77b2JjYzEYDDRq1Ih///vfXp1K/U3CVgghhPAxaeMRQgghfEzCVgghhPAxCVshhBDCxyRshRBCCB+TsBVCCCF8TMJWCCGE8DEJWyGEEMLHJGyFT8yaNYu6deuqP5s3b/aaP2HCBHXe0qVL/VRKF8+yrly50q9luRE7duxg4MCBtGrVSj2eorinZ9euXdXtFbWzZ88ya9YsZs2aVezuP+pvvnzdxc0nwzWKm+KDDz7gnnvukeH7fCQ+Pp4xY8YUmzuc5Ne5c+f48MMPAejfvz/du3f3c4mE8A355BM3xdGjR1mzZo2/i1GspKSkFNm2PG8l1r59ew4cOMCRI0ckvPygpH3hETeH1GyFz+l0OhwOB7NmzeK+++7DYDDkuOyQIUOIiooCXDdbqFq1KuBq6nXXgKZOncqAAQMAV1PbuXPnANddkN566y327NlDmTJlGDJkCCNGjOCnn35i5syZ/PPPP1StWpVRo0Zx//33Z7t/h8PB7Nmz+frrr7l8+TI1a9Zk7Nix9OrVy2u5M2fO8Omnn/LLL79w6dIljEYj9evXZ/DgwfTu3VtdbufOnTzxxBOAq+bWunVrPv/8c06ePMmoUaMYN25cjq+FzWbjyy+/5IcffuDEiROkp6dTqVIlOnXqxKhRo9QbUHi+ZgC//PILjRo1yvIaZhYdHc0HH3zA3r17iYuLw2w2ExYWRr169Xjqqado3LhxlnXOnz/P//73P37++Wf0ej133XUX//3vfylbtqy6jKIofP3113zzzTccO3aMtLQ0ypUrx1133cXTTz9N9erVsy33qlWrWLVqlfpaTZs2jUuXLjF9+nR+//13rl69isFgIDQ0lLp16zJo0CA6dOiQ4+sHqE2wVapUYcaMGURGRvLnn39iNBrp1q0bL774IiEhIV5lX7VqFd988w1HjhwhNTWV8uXL07lzZ0aPHu110w/P8i9ZsoTFixezY8cO4uPjOXLkSK7lio6OZv78+fz+++9cvnwZk8lE1apVeeSRR3IdUzotLY3XXnuNgwcPcunSJRITE9Hr9dxxxx10796dkSNHEhgYqC6/detW5s+fz5EjR0hKSqJUqVJUrFiRhg0b8tJLL6n3I165ciVLly4lOjqatLQ0LBYLVapUoVGjRrz66qsy/noRkLAVPte7d282bNjAmTNnWLFiBYMGDfLJfgYPHkxsbCwAFy9e5H//+x/79+9n/fr16r0tjx8/zn/+8x+qVq1KixYtsmxj1qxZXLp0SX1+9OhRnnvuOaZPn859990HwF9//cWwYcNISkpSl7Pb7ezevZvdu3dz8OBBXnjhhSzb/umnn9QwyUtaWhrDhw9nz549XtNPnTrFl19+ydq1a1m6dCnVqlXL1/ay2/7QoUOJiYlRpyUmJpKYmMjJkyfp0KFDtmH7yCOPeK2zdu1aEhISmD9/PuAKq/Hjx6t3ZXE7f/48K1euZN26dSxYsICmTZvmq5yjRo3i0KFD6nO73U5ycjJnz57lzjvvzDNs3a5du8aQIUPU1oSUlBRWrlzJoUOHWL58OUajEUVReOGFF/jhhx+81j137hxLlixh48aNLFu2LNsvL2PGjOHatWv5Ksu2bdsYO3YsNpvN67gOHz7ML7/8kmfYZu5XYLfbOXr0KEePHuWvv/5SfxcHDhxg7NixpKenq8vGxcURFxfH4cOHGTVqFKVLl2bTpk1MnDjRa5uxsbHExsayf/9+Xn75ZfR6iYobJc3IwucqVarEY489BsCcOXN81szWokULfv/9d95//3112rp16+jTpw9RUVFERESo01evXp3tNtLS0li8eDF79uxh/PjxgCtApk2bhsPhAGDy5MkkJSVhsVhYsGAB+/fv56effqJVq1YAfPrppxw9ejTLtq9du8bQoUP59ddf2blzJw8++GCOx/Lll1+qQdugQQPWr1/P77//rtbIr1y5wpQpU9Rlv/jiC3Xd/v37c+TIEY4cOZJjrfb48eNqaA4ZMoQ//viDPXv2sGbNGiZPnqzWPjOrWbMm27ZtY+3atWptdseOHeq21q1bpwZtlSpVWLlyJbt372bkyJEAJCcnM3ny5DzLPW3aNOLi4tSg7dWrF3v27GHfvn2sXbuWN998M9svAzlJTk7m/vvvZ+fOnXz//ffUqFEDgEOHDqnhtWHDBjVoBwwYwI4dO9i/fz/vvfceADExMbz77rvZbt9sNrNo0SL+/PPPHN9b4Hp/TZw4UQ3ahx56iC1btrB3716WLFlC586dcz0Os9lMZGQkmzZtYu/evRw4cICNGzdSv359wPW7cNeqd+3apQbt+++/z4EDB/jtt99YtmwZY8aMUWvAv//+u7r9r776igMHDrBjxw4WLlzIiBEjpJ9FEZGvK+KmePrpp1mxYgWXL1/myy+/9Mk+nn/+eUJCQujatavX9HHjxlG6dGm6detGZGQkgNr0nNmjjz6qhubTTz/N0qVLuXTpEpcuXeL48eOYzWY1SK1WK8OGDcuyDUVR2LFjh3rvYbfq1aszYcIE9cOrTJkyOR6LZ+/tsWPHquHw8ssvs2bNGhRF4ZdffiEtLQ2TyZTzi5KDSpUqYTAYsNvtbN++ncDAQGrWrMmdd97J448/jk6ny3a9yZMnU7FiRQBatWrF+vXrAdfrWa5cOa9yDxs2jIYNGwIwfvx4VqxYQVxcHMeOHeP06dN51sotFgtlypQhLi6OvXv38tFHH1GrVi3Cw8N58MEHMRqN+T5evV7PhAkTCAwMpEyZMjz55JO88sorgKvZfeDAgWzcuFFdfuXKldn2TP/555+z3f748eNp3bo1gBp82dm7dy9Xr14FoFq1arz55pvqa92yZUtatmyZ63EYjUbS0tJ46aWXOH78OAkJCTidTq9loqOjqVu3LnfccYc6bcmSJZw8eZKaNWtSv359nn32WXWe53Jz586lRYsW1KpVi4YNG/Liiy/mWh6RfxK24qYIDQ1l2LBhzJ49m3nz5qmBlhvPuz96NoXlxF0bM5vNXtPdtTvPD2fPJjxPlStXVh9rNBoqVaqkNivHxsbm+wM+uybF+vXr57uW4P5ABlcN0a106dIEBweTkJBAeno6cXFxVKhQIV/b9BQaGsrbb7/Nu+++y6lTp/j444/VeeXKlSMyMpK77rory3rh4eHq44CAAPVxWlpalnJ7vpZ6vZ6KFSsSFxcHuGrmeYWtVqtl+vTpvPrqq5w5c4bPPvtMnWexWHjttdfo06dPvo43JCTE61ymZ9ncZfYse06Sk5Ox2WxZ3gfuc+R58WyCDw8Pz/FLTU4+++wz3nnnnVyXcbccde/enSeffJKlS5eya9cudu3apS7TsGFD5syZQ4UKFXjsscf4+++/+fHHH9m6dStbt25Vl2vXrh0fffQRQUFBBSqnyErCVtw0I0aMYMmSJcTFxXn9QXvy/BDzbG4+ffp0ntvP6bxSQc43nT9/Xn2sKAoXLlxQn4eGhnoFea1atVi7dm2228nuNtGe4ZSXsmXLcurUKcBVa6xXrx7gusQnMTERcB1XbrXjvNx///3069ePf/75h5MnT3L8+HHmzp1LTEwMr732WpbzroBX57bsOs14dpTyfC0dDgcXL15Un4eFheW4DU/t27dn06ZNnD59mhMnTnDixAnmzZtHTEwML7/8Mvfee2++AuvatWskJyergetZNneZPcs+ffr0bINcUZRsy5z5C15OPDtY/fPPPzidzgI103733Xfq48mTJ/PII49gNpsZN24cGzZsyLL8Sy+9xP/93/9x5MgRzp49y65du1i8eDF///03H330EW+88QZGo5F3332X1157TV3up59+4vvvv+e3335j8eLF/Pvf/853GUX2pDFe3DTBwcHquTv3+c/MPGtx7kDes2fPTRvwYMWKFezdu5fExETmzp2r1morVKhA7dq1qV69uto8/M8///DOO+9w+fJl7HY7Z86cYfHixfTr1y/HZur88mwK/+ijjzh16hRxcXG8/fbbapC3b9++UE3I4KqlT506laioKIKDg+nUqRO9e/dWe6d6fskobLkXLFjAoUOHSExMZMaMGWqttnbt2mqt1vPLwsmTJ0lOTvba3uuvv8727dsxGAy0a9eO3r17U758ecBVy3RvMy/p6em88847xMfHc/z4ca9acvv27QHo0aOHOu29994jKiqKtLQ0EhIS2LlzJxMnTuT111/P92uRnRYtWqihfurUKV555RXOnz9PcnIyf/75J1999VWu63t+sQgMDESj0bBp0yZ++umnLMtGRUUxd+5cjh07RpUqVejevbvXpWDu3/H69etZuHAhZ8+eJTw8nF69etGuXTt1Oc8vJqLwpGYrbqrBgwezcOFCLl++nO38Bx54QP3Aee+99/j4449JTEwkMDAwx6bfomQwGNTOXJ5eeukl9YPurbfeYvjw4SQmJvLZZ595fXAXlSFDhrB582b27dvH33//Tc+ePb3mly1blkmTJhV6+zabjQULFrBgwYJs5+fVUScnvXv3Zu3atWzYsIFz585l6QQWEBDAm2++qT6vXr06oaGhxMbGsm/fPpo3bw5cv7zrq6++YsmSJdnuq2HDhl610dwEBgby/fffs2zZMq/p9evXVy8j69mzJ3379uX777/n3LlzDBkyJMt2+vfvn6/95cRkMjF16lS1N/KKFStYsWKFOr9bt248+uijOa7fq1cvDhw4ALhqtpMnT0ar1VK1atUsrT8XLlzg/fff9+ow6Mn9O46OjmbGjBk57rOw7wXhTWq24qYym82MGTMmx/ktW7YkMjKS2rVrYzQaCQ0N5YUXXmDo0KE3pXzjxo3jueeeo3LlyhgMBurUqcOMGTO8mhSbNGnCd999x6BBg6hevTpGo5HAwEBq1KjBvffey7Rp09TaV2GZzWa++OILXnjhBRo0aEBAQAAGg4E77riDxx9/nFWrVqmdpgrDYrHw5JNP0rx5c8LCwjAYDJhMJmrXrs2oUaPyPC+YE41Gw4wZM3j99ddp1qwZQUFB6PV6KlWqRP/+/Vm1apXXJVcmk4kPPviAJk2aeJ1Tdfv3v/9NmzZtKFeuHAaDAYPBQLVq1Xj88ceZN29evssVEhLC4sWLufvuuwkICMBisdC/f38+++wz9dSFRqMhMjKSd999l7Zt21K6dGn0ej3lypWjadOmPP300zz55JOFel08de7cmdWrVzNgwACqVKmCwWAgKCiIevXqqbXsnIwYMYJnn32WKlWqYDQaqVevHh9++GG2HasaNWrEww8/TJ06dShTpgw6nY6goCCaNWvGm2++yeDBgwHXedkHHniAWrVqUapUKbRaLaVLl6ZNmzZ8+OGH3HPPPTd8zAI0SnYnl4QQ4hbgOajFli1b/FwacTuTmq0QQgjhYxK2QgghhI9JM7IQQgjhY1KzFUIIIXxMwlYIIYTwMQlbIYQQwsckbIUQQggfk7AVQgghfEzCVgghhPAxCVshhBDCxyRshRBCCB+TsBVCCCF87P8BZSj5fy9Z+goAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 500x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_few_shot(protomaml_accuracies, name=\"ProtoMAML\", color=\"C2\", ax=None)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch-lightning-env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "37b693a6f780c8511f891a639fe901e9eb59e5ec0a37b86f170d6f7c2ce5e026"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}