12.A2C.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Advantage Actor Critic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "gym.logger.set_level(40)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from timeit import default_timer as timer\n",
    "from datetime import timedelta\n",
    "import os\n",
    "import glob\n",
    "\n",
    "\n",
    "from baselines.common.vec_env.dummy_vec_env import DummyVecEnv\n",
    "from baselines.common.vec_env.subproc_vec_env import SubprocVecEnv\n",
    "\n",
    "from utils.hyperparameters import Config\n",
    "from utils.plot import plot\n",
    "from utils.wrappers import make_env_a2c_atari\n",
    "from agents.BaseAgent import BaseAgent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "port=8097\n",
    "log_dir = \"/tmp/gym/\"\n",
    "\n",
    "try:\n",
    "    os.makedirs(log_dir)\n",
    "except OSError:\n",
    "    files = glob.glob(os.path.join(log_dir, '*.monitor.csv'))\n",
    "    for f in files:\n",
    "        os.remove(f)\n",
    "\n",
    "config = Config()\n",
    "\n",
    "#a2c control\n",
    "config.num_agents=16\n",
    "config.rollout=5\n",
    "\n",
    "#misc agent variables\n",
    "config.GAMMA=0.99\n",
    "config.LR=7e-4\n",
    "config.entropy_loss_weight=0.01\n",
    "config.value_loss_weight=0.5\n",
    "\n",
    "# Number of updates in 10000000 frames\n",
    "config.MAX_FRAMES=int(1e7 / config.num_agents / config.rollout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rollout Storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RolloutStorage(object):\n",
    "    def __init__(self, num_steps, num_processes, obs_shape, action_space, device):\n",
    "        self.observations = torch.zeros(num_steps + 1, num_processes, *obs_shape).to(device)\n",
    "        self.rewards = torch.zeros(num_steps, num_processes, 1).to(device)\n",
    "        self.value_preds = torch.zeros(num_steps + 1, num_processes, 1).to(device)\n",
    "        self.returns = torch.zeros(num_steps + 1, num_processes, 1).to(device)\n",
    "        self.action_log_probs = torch.zeros(num_steps, num_processes, 1).to(device)\n",
    "        self.actions = torch.zeros(num_steps, num_processes, 1).to(device, torch.long)\n",
    "        self.masks = torch.ones(num_steps + 1, num_processes, 1).to(device)\n",
    "\n",
    "        self.num_steps = num_steps\n",
    "        self.step = 0\n",
    "\n",
    "    def insert(self, current_obs, action, action_log_prob, value_pred, reward, mask):\n",
    "        self.observations[self.step + 1].copy_(current_obs)\n",
    "        self.actions[self.step].copy_(action)\n",
    "        self.action_log_probs[self.step].copy_(action_log_prob)\n",
    "        self.value_preds[self.step].copy_(value_pred)\n",
    "        self.rewards[self.step].copy_(reward)\n",
    "        self.masks[self.step + 1].copy_(mask)\n",
    "\n",
    "        self.step = (self.step + 1) % self.num_steps\n",
    "\n",
    "    def after_update(self):\n",
    "        self.observations[0].copy_(self.observations[-1])\n",
    "        self.masks[0].copy_(self.masks[-1])\n",
    "\n",
    "    def compute_returns(self, next_value, gamma):\n",
    "        self.returns[-1] = next_value\n",
    "        for step in reversed(range(self.rewards.size(0))):\n",
    "            self.returns[step] = self.returns[step + 1] * \\\n",
    "                gamma * self.masks[step + 1] + self.rewards[step]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ActorCritic(nn.Module):\n",
    "    def __init__(self, input_shape, num_actions):\n",
    "        super(ActorCritic, self).__init__()\n",
    "\n",
    "        init_ = lambda m: self.layer_init(m, nn.init.orthogonal_,\n",
    "                    lambda x: nn.init.constant_(x, 0),\n",
    "                    nn.init.calculate_gain('relu'))\n",
    "\n",
    "        self.conv1 = init_(nn.Conv2d(input_shape[0], 32, kernel_size=8, stride=4))\n",
    "        self.conv2 = init_(nn.Conv2d(32, 64, kernel_size=4, stride=2))\n",
    "        self.conv3 = init_(nn.Conv2d(64, 32, kernel_size=3, stride=1))\n",
    "        self.fc1 = init_(nn.Linear(self.feature_size(input_shape), 512))\n",
    "\n",
    "        init_ = lambda m: self.layer_init(m, nn.init.orthogonal_,\n",
    "          lambda x: nn.init.constant_(x, 0))\n",
    "\n",
    "        self.critic_linear = init_(nn.Linear(512, 1))\n",
    "\n",
    "        init_ = lambda m: self.layer_init(m, nn.init.orthogonal_,\n",
    "              lambda x: nn.init.constant_(x, 0), gain=0.01)\n",
    "\n",
    "        self.actor_linear = init_(nn.Linear(512, num_actions))\n",
    "\n",
    "        self.train()\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        x = F.relu(self.conv1(inputs/255.0))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = x.view(x.size(0), -1)\n",
    "\n",
    "        x = F.relu(self.fc1(x))\n",
    "\n",
    "        value = self.critic_linear(x)\n",
    "        logits = self.actor_linear(x)\n",
    "\n",
    "        return logits, value\n",
    "\n",
    "    def feature_size(self, input_shape):\n",
    "        return self.conv3(self.conv2(self.conv1(torch.zeros(1, *input_shape)))).view(1, -1).size(1)\n",
    "\n",
    "    def layer_init(self, module, weight_init, bias_init, gain=1):\n",
    "        weight_init(module.weight.data, gain=gain)\n",
    "        bias_init(module.bias.data)\n",
    "        return module"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(BaseAgent):\n",
    "    def __init__(self, static_policy=False, env=None, config=None):\n",
    "        super(Model, self).__init__()\n",
    "        self.device = config.device\n",
    "\n",
    "        self.noisy=config.USE_NOISY_NETS\n",
    "        self.priority_replay=config.USE_PRIORITY_REPLAY\n",
    "\n",
    "        self.gamma = config.GAMMA\n",
    "        self.lr = config.LR\n",
    "        self.target_net_update_freq = config.TARGET_NET_UPDATE_FREQ\n",
    "        self.learn_start = config.LEARN_START\n",
    "        self.sigma_init= config.SIGMA_INIT\n",
    "        self.num_agents = config.num_agents\n",
    "        self.value_loss_weight = config.value_loss_weight\n",
    "        self.entropy_loss_weight = config.entropy_loss_weight\n",
    "        self.rollout = config.rollout\n",
    "        self.grad_norm_max = config.grad_norm_max\n",
    "\n",
    "        self.static_policy = static_policy\n",
    "        self.num_feats = env.observation_space.shape\n",
    "        self.num_feats = (self.num_feats[0] * 4, *self.num_feats[1:])\n",
    "        self.num_actions = env.action_space.n\n",
    "        self.env = env\n",
    "\n",
    "        self.declare_networks()\n",
    "            \n",
    "        self.optimizer = optim.RMSprop(self.model.parameters(), lr=self.lr, alpha=0.99, eps=1e-5)\n",
    "        \n",
    "        #move to correct device\n",
    "        self.model = self.model.to(self.device)\n",
    "\n",
    "        if self.static_policy:\n",
    "            self.model.eval()\n",
    "        else:\n",
    "            self.model.train()\n",
    "\n",
    "        self.rollouts = RolloutStorage(self.rollout, self.num_agents,\n",
    "            self.num_feats, self.env.action_space, self.device)\n",
    "\n",
    "        self.value_losses = []\n",
    "        self.entropy_losses = []\n",
    "        self.policy_losses = []\n",
    "\n",
    "\n",
    "    def declare_networks(self):\n",
    "        self.model = ActorCritic(self.num_feats, self.num_actions)\n",
    "\n",
    "    def get_action(self, s, deterministic=False):\n",
    "        logits, values = self.model(s)\n",
    "        dist = torch.distributions.Categorical(logits=logits)\n",
    "\n",
    "        if deterministic:\n",
    "            actions = dist.probs.argmax(dim=1, keepdim=True)\n",
    "        else:\n",
    "            actions = dist.sample().view(-1, 1)\n",
    "\n",
    "        log_probs = F.log_softmax(logits, dim=1)\n",
    "        action_log_probs = log_probs.gather(1, actions)\n",
    "\n",
    "        return values, actions, action_log_probs\n",
    "        \n",
    "\n",
    "    def evaluate_actions(self, s, actions):\n",
    "        logits, values = self.model(s)\n",
    "\n",
    "        dist = torch.distributions.Categorical(logits=logits)\n",
    "\n",
    "        log_probs = F.log_softmax(logits, dim=1)\n",
    "        action_log_probs = log_probs.gather(1, actions)\n",
    "\n",
    "        dist_entropy = dist.entropy().mean()\n",
    "\n",
    "        return values, action_log_probs, dist_entropy\n",
    "\n",
    "    def get_values(self, s):\n",
    "        _, values = self.model(s)\n",
    "\n",
    "        return values\n",
    "\n",
    "    def compute_loss(self, rollouts):\n",
    "        obs_shape = rollouts.observations.size()[2:]\n",
    "        action_shape = rollouts.actions.size()[-1]\n",
    "        num_steps, num_processes, _ = rollouts.rewards.size()\n",
    "\n",
    "        values, action_log_probs, dist_entropy = self.evaluate_actions(\n",
    "            rollouts.observations[:-1].view(-1, *obs_shape),\n",
    "            rollouts.actions.view(-1, 1))\n",
    "\n",
    "        values = values.view(num_steps, num_processes, 1)\n",
    "        action_log_probs = action_log_probs.view(num_steps, num_processes, 1)\n",
    "\n",
    "        advantages = rollouts.returns[:-1] - values\n",
    "        value_loss = advantages.pow(2).mean()\n",
    "\n",
    "        action_loss = -(advantages.detach() * action_log_probs).mean()\n",
    "\n",
    "        loss = action_loss + self.value_loss_weight * value_loss - self.entropy_loss_weight * dist_entropy\n",
    "\n",
    "        return loss, action_loss, value_loss, dist_entropy\n",
    "\n",
    "    def update(self, rollout):\n",
    "        loss, action_loss, value_loss, dist_entropy = self.compute_loss(rollout)\n",
    "\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.grad_norm_max)\n",
    "        self.optimizer.step()\n",
    "\n",
    "        self.save_loss(loss.item(), action_loss.item(), value_loss.item(), dist_entropy.item())\n",
    "        #self.save_sigma_param_magnitudes()\n",
    "\n",
    "        return value_loss.item(), action_loss.item(), dist_entropy.item()\n",
    "\n",
    "    def save_loss(self, loss, policy_loss, value_loss, entropy_loss):\n",
    "        super(Model, self).save_loss(loss)\n",
    "        self.policy_losses.append(policy_loss)\n",
    "        self.value_losses.append(value_loss)\n",
    "        self.entropy_losses.append(entropy_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updates 10300, Num Timesteps 824080, FPS 1628,\n",
      "Mean/Median Reward -20.2/-20.5, Min/Max Reward -21.0/-18.0,\n",
      "Entropy 1.78179, Value Loss 0.00865, Policy Loss 0.00615\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABNwAAAFoCAYAAACMr+bUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3XmUpGV59/HvVb3MwAwM4CAgAoOK4BZFQQ0uKOACakzcY6JiNMRE1ASiCaKAa1RQ37gkuKMiUcQtohJFHIhCgAEExA3BQXaHbWDW7q663j+ep7qrq6u6q2qqu5mZ7+ecPtXPfld1ec7447rvKzITSZIkSZIkSf1Rme8BSJIkSZIkSVsSAzdJkiRJkiSpjwzcJEmSJEmSpD4ycJMkSZIkSZL6yMBNkiRJkiRJ6iMDN0mSJEmSJKmPDNwkSZKkBhHx3ojIiPhsl9edXl73jtkamyRJ2jwYuEmSpE0WEaeVQUPzz70R8fOIODkiHjzf4+yHpvf3pmnOG2w47xl9fP4z2nzWzT8/79cztXWJiL0iYk3Dd+mp8z0mSZI2N4PzPQBJkrRFGQXuKn8PYGfgseXP6yPiBZn50/ka3Cw4PiI+l5nr5un5dwDVaY5pbt0C/IbN/7P/BLBovgchSdLmzAo3SZLUTxdm5q7lzy7AYuDVwD3ADsDXI2KbeR1hf+0CtK1ymwMHNnzezT+HzeO4tkqZ+bbM3C8zT53vsfQqIl4MPB+4eL7HIknS5szATZIkzZrMXJeZXwbeXO7aFfjzeRxSP/2gfH1bRGw/ryOR+iAitgP+HbgXeOs8D0eSpM2agZskSZoLZwK18vcnNB6IiF0i4sMR8euIWBcRqyPikog4NiIWtLpZw5pxJ0XEQET8Y0RcWV5/V0ScHREHTDegiHhKRHyvPH9tef0/RkSl8f7T3OJTwI3ATsCxHX8Sk8ewffkerizXzFoTEVdFxLsiYkkv95zmWeML+kfEwoh4Z/ms+lpdi8vzFkfEyyPijIi4JiLuiYj1EXFtRJwaEQ9rc//GNeseHBGPKO9xa/n5Xh4Rf9VwfkTEGyLisnIMd5bnT7vWX0TsHRGfiIjfln/veyNiRUS8LSK2bXPN9hFxYjmG+yJiY0TcEhGXRsSHIuKRXX6W7yjf5/qIeH6rz7jp/IeV+8fK7adFxPcj4o7yPVwREf8QEV3/27z8/mRE/N8M572qPO/WiBhoc9r7gd2BE4Bbux2LJEma4BpukiRp1mXmxoi4A3ggMF4NFhFPpKgU26ncdR8wDBxY/rwqIp6dmX9sc+tB4GzguRTrx20EdgSeBxwaEYdk5kXNF0XEq4EvMPEfH+8BHgl8FHg6RYXPTDYC7wE+DfxTRHwsM+/s4Lr6GB4GnAvsVe6qrwP3mPLnyIg4LDOv7fSeHdoW+ClF8DkCrG86/nqKz6HuPorP6WHlz19FxJ9l5k+mecafAp+nmFK8GtgG2B84PSKWAh8Hvgq8tBzDKMV34C+BP42Ix2fm3c03jYiXAl8G6kHs+vL3J5Q/r4yIZ2XmqoZrdgQuAvYtd9XKMe0C7AYcUI6ho86iEfFh4BhgDTDT59Dq+pcBZwADFN+7YeBxwCeBQyLi5ZnZbl2+Vr4CnAg8KSIekpnXtznvL8vXr7W6fxlQ/wNwJcUabnt3MQZJktTECjdJkjTroli3bedy855y347AtymClquBJ2bm9hQhzUuBuymaLXxlmlu/EXgi8HJgcWZuV17zC2AhxfS45rHsB3yG4t9B3wf2zswdKYLANwMvAF7Y4Vv7AvA7YDvgXzq8hogYBr5BEbbdCDyb4n0vBg4D/gDsCXwr2lT5bYI3UYQpL6P4zHYAHgJsKI/fAXyMIjTbofybbEMRSH61HOMZMf1afJ8Bfkzx2e5AEYJ+tjz2XuDdwHOAv6L47LYDDgZuB5bRYjpjRDyZiaDq34A9M3NbigDxIGAFxd/+tKZL/4kibPsjcASwIDN3ovh+7AscB/x+mvdSf34lIj5LEbbdBRzabdhG8Z37LHAOsKz83i0B/hVI4MV0WS1ZBrKXlZt/2eqciHgA8Kxy84wWxwcoguMA/r7LwE+SJLVg4CZJkubC6yj+zzxMLMZ+NEWF0T3AszPzUoDMrGbmWcAryvMOi4hD2tx3B+CFmXlmZo6U118FHFkePzAi9mq65jiKqqJfAH+RmSvL69Zn5seB48v7zigzx4B31d9PROzWyXUUAeGfAGPAEZn5o5zwY4pgaBR4FEUo1c6lEXFbm59268otBl6WmV/PzNHyffy+fC9k5umZ+ZbM/L/MXF3uq2Xmr4BXAssp1uJ70TTjuhV4ccNnuxr4O4pgazHFZ/z3mXlGZo6U7/sCir8NwEta3POjFBWNx2Tm2zPzxvLe1bKK8bnAbcAREfG4huueXL5+KDN/0PA+RzPzt5n5gcz83DTvhYgYoggbX1c+4xmZecl017S7FcVn8KLMvKEcx9rM/CDFdE6A42YIM1uph2ivbHP8pRSf3XVtxv0WigrEz7eqCJUkSd0zcJMkSbOiXKNrWUT8M/ChcvcNwHfL3+uhymcz87bm6zPzhxRTAaGoxmrlfzPzpy2uvQy4qdx8VMOYKkw0bfh/9ZCuySeAtW2e18oZwDUUVWDHd3hN/b1/OzN/0XwwM68Bzio32713gKUUUyNb/bT7d97lZajXtcxM4Hvl5lOmOfXk5iqpzKwB9YqwG4D/anFdfVwPi4iF9Z0RsS9FcLaWohKr1djuBP6n3HxWw6H69OBOw9BJynXh/psitLoBeFpmXt3LvUqntPnenUIxtXUH4NAu7/lViqmyj4yIP2lxvF751qq6bQ+K0PguuqjSlCRJ0zNwkyRJ/XRwfeF8igDg98DJFGHUrcCfZ+ZIOaXy0eU1003LO698fXyb45dOc+3N5euODfsewsQaclOCOig6qzIxRW9GZZB0Yrn5ty0q6lqpv59Nee9QTNmMNj/3tLlmxgqmiNizbCZwWRRNE6oNf9eTy9MeNM0t2gVS9bX4rinDu2a314dAMdWy7qDydQFwQ7uqPiaCzD0arv1++fpPEfHFiHhuFN04O7GEIsR7LvAb4KmZ+bsOr21neaud5d/rynJzur95q2tvAc4vNydVuZVNKJ5Wbk4J3CjW01sM/Gs3axBKkqTpGbhJkqR+GqUITW6nmHp3HfAj4G3AozLz5+V5OzHx75Cbm2/SoF6ltnOb4/dNc219TbKhhn1LG36frgvjLdMca+WbwOUUU1VP6OD8+vvp5L0/ICJimvO6tWq6g+X03V9SrKP2eIqA8j4m/q71irFF09ym3Wdb7fA4TP671avTBmlf0bdLw5gau5V+Afgcxfft1RRNOu4pO5aeFBG7TvM+XgI8laLy7LmZedM053Zquu9W/fsw/n0vx9kqYPzHpmvrYdormr4vr6AIMK/IzF83XhARf0GxXuHFTKyxJ0mS+sDATZIk9dOFmblr+bNbZj4sM5+dmSe36jpZ6ndTgOn0M7gaV1ZrvbPcfE1EPLzDS+fyvde1XRC/bNDwZYrg6ocUYdM2mblD/e9KEZ7CLH2WbdT/zXrpNBV9jT+vr19Yrg/3eoo1895DUQk2QrFm2YnAtdOsEbicIjgeBj7bOM11lrT6TB9I63BxcdN5Z1G8r72YqAiENtNJy1DuYxTNGt4KLIqIxfUfJoeW25T7Z/v9S5K0xTBwkyRJ8+EuiimnUAQE7Ty4fJ22KqsLjfeZbk2vrtf7yszvAz+j6KL5rhlOr4+jk/d+Z5vpl7PhKRRTRe+gaEbxs8zc2HTOLnM0lkb1qab7lh01u5aZV2fmCZn5DIp10v6MYu29xcCX2tz3Oor11FaVr98sp0Nvik6+d+Pf08x8cJtQ8b2NF5ZTUn9Qbv4lQBn8Pp4iVPtq07MGKL5jAVxAUcXY+HNlw7k/LPed3fG7lCRpK2fgJkmS5ly5aHy9WcAzpzm1Xnl0eZ8efT0TUyKf2uqEskPkE3q8/zvK15cDj5nmvPr7mcv33ol6yPfrzNzQ5pzD5mowDerrzm1P9w0FpsjMjZn5XYq/E8DuwEPbnPtLivd8F3A48PWya2mvDm61MyKWAI8tN3v9m9er2F4WEYNMVLdd0KfpsJIkqUMGbpIkab7Uu3AeGRFTqn4i4tnAn5abZ/bjgWWDg++Um29pE5z8A1On63V6/+UUnTaDYvpiO/X3fnhE7N98MCIexUQDgL689w6tLl/3bVXJFRFHMLEA/5wpO7muKDc/VHYObSkitm0c+wwVaesbfm87vTczr6LofHoPRWXcGb1W2gH/3OZ7d0w5hnuAc3u893cpKtF2pggm23Ynzcyx6ablAvs0nP60cv98hK2SJG2WDNwkSdJ8+QTF4vnbAOdExAEAETEQES9mYgrcuZl5Xpt79OLfKNa6egzwjXpX0YhYGBFvLI+36/DZiePL1+dNc87XgKvK378dEYfVF7qPiEMpOmsOUUx5/MomjKVbP6UIoXYGvlhvKBAR20TE31KEf/PVyfJoir/bY4ELIuKQeugVEZWIeHREnEAxDfSBDdf9JCL+PSKe1rgGWUQ8Gvh8uXkTRaOItjLzcuA5FBWSLwG+HBHd/ls6KSrpvhERe5bjWBQRb2WiOvLfpqkunP7mmeuBb5eb7wf2pWhkclbbiyRJ0qwwcJMkSfOibKLw58DdFAvaXxoR9wJrKAKCHSlCqb/q83N/BbyBIvx4AbAyIu6iCFI+QRFY/Hd5evP6ZZ3c/2JmWOuqnFL7YuAGYE+KTq5rImItRXXTnsAfgBe1WENt1mTmnUwEP68Abo2Ieyg+m08DvwHe2+by2R7bxRSf2X0UU35/DKyLiDsoOtJeTbF23q4Uf9u6JcCbKdYpWxsRd0VE/fyDgbXAqzKzbTOJhjFcAhxB8R39S+BzXXaQrQF/W97jhvJ7dw/wIYp/l38D+HAX92ulXs32+PL1nMy8axPvKUmSumTgJkmS5k0ZYDwS+CjwW4qqrjGK6YNvBZ6UmX+ched+AXg6cA7FNMoFFBVOb6EImpaUp/Za6fZOJoc+rcbwO4pqrXczsZ4d5e/vAf4kM3/b4/N7lpkfAV5KsW7aemAQ+DXFe3oKRdg0LzLzbIqpju8HrqAI2nagCAR/Vo5xv8y8ueGy1wInUXQc/QNFRWUN+BVFl85Hl1OBOx3Dz4DnA+uAI4FTuwndMvNrFOvz/YDiOzIG/Bx4I/CyToK/GfwIaPzfzJTppJIkafbF3DW9kiRJuv8rw5MbgD2AZ3YTxkitRMTDgGuBamYOzvd4JEnS7LPCTZIkabJXUIRt9wKXzPNYJEmStBnyv7BJkqStTkS8nWItsG8DN2dmLSJ2BF5N0TQB4D8yc918jVGSJEmbL6eUSpKkrU5EnM5EM4YRioXzdwDqa3GdC7yg126RUiOnlEqStPXZrKeURsQ+EfEvEXFeRNwYESMRcXtEfCcinjnDta+JiEsiYk1ErI6I5RHx/B7H8fzy+tXl/S6OiNf09q4kSdIc+A/gPyk6Vd4LbAfcBfwEeB1wuGGbJEmSerVZV7hFxFeBl1N0FfspxT+U9wX+DBgA3pKZH2tx3SnAscBNwFnAMMV6LTsBb8rMT3QxhqOBjwN3Al+j+K/kLwEeDHw4M/+51/cnSZIkSZKkzc/mHrgdCVyZmVc07T+YoiV6Assy89aGYwdRtI2/DjgwM+8u9y8DLgMWUbSTX9nB85cBv6aYhvKE+jXlGjCXAg8FDsrMizp5P0uXLs1ly5Z1cqokSZIkSZI6cNlll92RmTvP5TM36zUkMvO0NvvPj4jlwLOAg4BvNBx+Q/n6vnrYVl6zMiI+CbwTeC1wYgdD+BtgAfDBxoAuM++OiPcDnyuf11HgtmzZMlasWNHJqZIkSZIkSepARNww18/crNdwm8Fo+TrWtP+Q8vWcFtf8oOmcmfTzXpIkSZIkSdoCbJGBW0TsBRwKrAMuaNi/CNgdWNM4zbTBteXrwzt81L7l62+bD5T3Xws8OCK27fB+kiRJkiRJ2sxtcYFbRCwAvkIx1fOkxmmjwJLydXWby+v7d+jwcZ3eb0mb40TEURGxIiJWrFq1qsPHSpIkSZIk6f5q3gO3iFgZEdnFz+nT3GsA+DLwFIqOoaf0OKx+dZKIme6XmZ/OzAMy84Cdd57T9fskSZIkSZI0C+4PTROuAzZ0cf4trXaWYdvpwEuBM4G/zqktWGeqOJupYq3ZamBped2dLY5vX77e2+H9JEmSJEmStJmb98AtMw/d1HtExCBwBkXYdgbw6systnjW2oi4Gdg9InZrsY7bPuXrlDXZ2vgNReD2cJo6kUbEbsAi4KbMXNfxm5EkSZIkSdJmbd6nlG6qiBgGzqII274EvKpV2NbgvPL1uS2OHd50zkz6eS9JkiRJkiRtATbrwK1skPAt4IXA54DXZmZthstOLV+Pj4gdG+61DHgjsBH4QtNzlkbEfhGxtOleXyjPP7q8vn7+jsDbm54nSZIkSZKkrcC8TyndRKcCRwB3ADcDJ0RE8znLM3N5fSMzL4yIjwDHAFdFxFnAMPByYCfgTZm5sukeRwMnAu8CTmq41+8j4q3Ax4AVEfE1YAR4CfBg4MOZeRGSJEmSJEnaamzugdve5etS4IRpzlveuJGZx0bEVRRB2lFADbgcODkzz+5mAJn58YhYCfwz8GqKqsFfAu/IzC92c6/5cNF1d/L4vXZgweDAfA9FkiRJkiRpixBTG3lqvhxwwAG5YsWKOXveb267j+f8vwv46yfvyXv//DFz9lxJkiRJkqS5EhGXZeYBc/nMzXoNN22aO9duBODa29fM80gkSZIkSZK2HAZuW7Fa2V5ioDJl3TtJkiRJkiT1yMBtK1YtpxMbuEmSJEmSJPWPgdtWrFYrArfK1M6ukiRJkiRJ6pGB21asWrPCTZIkSZIkqd8M3LZi9SmlVrhJkiRJkiT1j4HbVqw2XuE2zwORJEmSJEnaghi1bMXOvvpWwCmlkiRJkiRJ/WTgthX73lVF4FbOLJUkSZIkSVIfGLiJ0aqJmyRJkiRJUr8YuImRam2+hyBJkiRJkrTFGJzvAWhuXbryLj51/vXssdM24/su+O0qRqs1huyeIEmSJEmStMkM3LYyx5z5c268a/2U/W876yo++vLHzcOIJEmSJEmStiyWNG1lbrp7atgGcOVN98zxSCRJkiRJkrZMBm5bmXYdSdePVOd2IJIkSZIkSVsoAzcBsHbj2HwPQZIkSZIkaYtg4CYAnrDXjvM9BEmSJEmSpC2CgZsAeOLeD5jvIUiSJEmSJG0RDNwEwGi1Nt9DkCRJkiRJ2iIMzvcANDeyXbeEkoGbJEmSJElSf1jhtpV4+7d+wd7Hfb/t8Y+f9zu+eflNczgiSZIkSZKkLZOB21bivy75Q8v9r3zSnuO/H3PmlXM1HEmSJEmSpC2WgdtWbvcdtuEBi4bnexiSJEmSJElbDAO3rdzwQIWhAb8GkiRJkiRJ/WLSspUbqAS1hoYKV954zzyORpIkSZIkafNn4LYVWjQ8MP77VTfdwx/v2zi+/cJP/mw+hiRJkiRJkrTFMHDbCn38lfuP/16pxDyORJIkSZIkactj4LYVGqxM/NkDAzdJkiRJkqR+MnDbCg0OTIRs2bB+W936kepcDkeSJEmSJGmLYuC2FVp5x7rx3//mqXtPOX7pyrvmcjiSJEmSJElblMH5HoDm3n0bRln5gee1PR7OMpUkSZIkSeqZFW5bodrUWaSTjM10giRJkiRJktoycNsKNK/JVmuxblujatXATZIkSZIkqVcGbluBf/zaFZO2H7fHDpO23/G8R0zatsJNkiRJkiSpdwZuW4ELf3fnpO1HP2jJpO3XP+0hk7arBm6SJEmSJEk9M3DbGjQ1QYgZ/upjtdrsjUWSJEmSJGkLZ+C2FWhuOlqZoQ2pFW6SJEmSJEm9M3DbCkRTwDZ93AbHnHnl7A1GkiRJkiRpC7dZB24RsU9E/EtEnBcRN0bESETcHhHfiYhnznDtayLikohYExGrI2J5RDy/y+c/LiJOioifRcSt5fNvjoj/iojHb9q765+ByuSIrVWF25UnPpu/2H/3uRqSJEmSJEnSFmuzDtyA9wAfAHYBvg98GPgZ8DzgvIh4c6uLIuIU4DRgN+AzwOnAY4DvRsTRXTz/VOBEYAHwTeCjwC+AVwAXR8RfdP+W+q85Xms1o3TJNkMcd8R+czIeSZIkSZKkLdngfA9gE50DfDAzr2jcGREHAz8CTo6Ir2fmrQ3HDgKOBa4DDszMu8v9JwOXAadExNmZubKD538F+OvM/F3T8/+KIsT7TER8LzNHen6HfdA8pXSmNdwkSZIkSZLUu826wi0zT2sO28r95wPLgWHgoKbDbyhf31cP28prVgKfpKhWe22Hz/94c9hW7v8KcC3wAIrKuXk1PDA5YGueYlpnECdJkiRJkrTpNuvAbQaj5etY0/5DytdzWlzzg6ZzZuP5c+5p++wMwA/e8jROe+2BbQO3pYsXjP9+34bRludIkiRJkiRpepv7lNKWImIv4FBgHXBBw/5FwO7AmsZppg2uLV8fvonPfxLwSOBmijXd5t2u2y/kEbttzyN2276j85unoUqSJEmSJKkzW1zgFhELKNZWWwC8rXHaKLCkfF3d5vL6/h024fk7Al8uN4/JzGqv9+qXsVq2rWprp5Y5S6ORJEmSJEnass37lNKIWBkR2cXP6dPca4Ai7HoK8DXglB6H1VPaVFbQ/TewD/ChzDyzg2uOiogVEbFi1apVvTx2RrXsPnCrVouP4KuX/IFf3nLvbAxLkiRJkiRpi3R/qHC7DtjQxfm3tNpZhm2nAy8FzqToHtocnNUr2JbQ2kwVcG2VYdv3gKcCH8nMf+nkusz8NPBpgAMOOGBWysrGaslgh4Hbg5Ys5JbVG9g4VuPeDaP86zevZs+dtuWCtz1zNoYmSZIkSZK0xZn3wC0zD93Ue0TEIHAGRdh2BvDqVlM5M3NtRNwM7B4Ru7VYx22f8vW3XT5/O4qw7WkUlW0dhW1zpVqrUekwcDv22fty7NevZONYldFqDYA/3LVuNocnSZIkSZK0RZn3KaWbKiKGgbMowrYvAa+aYd2088rX57Y4dnjTOZ08fwnwQ4qw7X33t7ANYKzaeYXbwqEBADaO1Vi9vuhU2uVsVEmSJEmSpK3aZh24lQ0SvgW8EPgc8NrMrM1w2anl6/Flg4P6vZYBbwQ2Al9oes7SiNgvIpY27d8ROBd4MnBiZr6j93cze7pZw23BYPGVuGvtCBtGi9xycGCz/ppIkiRJkiTNqXmfUrqJTgWOAO4AbgZOiJgSLC3PzOX1jcy8MCI+AhwDXBURZwHDwMuBnYA3ZebKpnscDZwIvAs4qWH/N4EDKNahq0TESUz17cz8eQ/vrW+6WcNtp8XDAHz3ylt44eN2B2DqUniSJEmSJElqZ3MP3PYuX5cCJ0xz3vLGjcw8NiKuogjSjgJqwOXAyZl5dg/PfyhFINfKSmBeA7dqLTtew23/PXYAYGigQq0M2qo1AzdJkiRJkqRObdaBW2Y+YxOu/SLwxQ7PPYnJlW31/ct6ff5cqnZR4RYR7LRomGotxwM38zZJkiRJkqTOuTjXVmCs2vkabgADlWCsVsOZpJIkSZIkSd0zcNsKXLLyrq7Cs1X3beS/LrlxvMJtvvzwmttY9q/f48Tv/GJexyFJkiRJktQNA7etQCVgu4Xdzx6e7wq3o758GQBfvOiG+R2IJEmSJElSFwzctnCZSS3hkbtt3/W1813hJkmSJEmStDkycNvCjVaL0Gx4sPs/tXmbJEmSJElS9wzctnAj1RrQW+D29m9d3e/hSJIkSZIkbfEM3LZwI2Nl4DbQ+Z/6MbsvAeDW1RtmZUySJEmSJElbMgO3Ldx44DY40PE1z330rrM1HEmSJEmSpC2egdsWbiJw6/xPPVCJ2RqOJEmSJEnSFs/AbTN134ZR/vvKW2Y8b6RaBboL3AZbBG7rRsY6H9ws+MOd6+b1+ZIkSZIkSZ0ycNtMHf+tX/Dm/7qCX99277TnbexhDbdWgdvTP/ST7gbYZ1+8aOW8Pl+SJEmSJKlTBm6bqV/eOn3QVlefUrqgmwq3FuHcHWtGOr6+X3bcdohXPXkvdth2iNGy26okSZIkSdL9nYHbZmrdxmKK50yVa72s4TY0cP9Yw220mgwNVBisVBir5XwPR5IkSZIkqSMGbpup+zYUgdv1q9ZOe95ItZemCa3PvfGudWTOXfC1ZuMYw4MVBitBbZrALTPZMFpte/za2++zQk6SJEmSJM0ZA7fN1H1lhdvrv7SCa25Z3fa8kR7WcGtX4fa0D/2Ez/zv9V2Msndryvc3Vq0xUIlpK9xOPf969nvnOdy5ZuOUY1fddA/P+ugFfOr862ZtrJIkSZIkSY0M3LYAN9+9vu2xXqaUDrapcAO4dOXdnQ9sE9xVrhm3bOkiBipBdZrA7RuX31Rcs3bqOnMry+6mna55J0mSJEmStKkM3LYA0y1v1suU0sFp1nCbbmpnP927YRSAnbdbwOAMFW716aKtmj2MloHjUBcVfpIkSZIkSZvCFGILMN26ahv7OKUUoDpHa7jV16jbbuEglRnWcKuHaoOVqeMeq7l2myRJkiRJmlsGbpuh21ZvmLQ9XdHZr2+9D4CFQwMd379d0wSA5b9ZNW2Dgn6pTxPdfuFQWeHWPji7pfw8biinjzYardrdVJIkSZIkzS0Dt83QzfdMDpamC6O2HS6Ctp23W9Dx/YeaKsUe++Alk7ZX3Te1OUG/1SvzHrHb9jOu4VZ32oW/n7JvrJxu+s7nP7K/A5QkSZIkSWrDwG0zNNZUtVUPp1oZqdbYpovqNpi6FtpRT3/opO36dM/ZtGG0yn67bsdAJToO3Fqt81bf180adpIkSZIkSZvCFGIz1LyO2rSB21ht2jXZWmlumrBwaPLX5L6yocFs2jBaHZ8GOzBD04TGa5rVp5QOTTNNVpIkSZIkqZ9MITZDzTNIv3ThyrbnjlZrXVd3NYdTzeu/femiG7q6Xy+KwK0Yx+A0FW7fLNd6A7h+1dopx8fGO5h2FzpKkiRJkiT1ysBtM9S8Ztu1f1zT9tzRaq2rDqUAe++8aNL2gqbA7ntX39rV/XryMYqcAAAgAElEQVSxYXRiKmwl2le4HXPmleO/H/zwnaccHy2va9XBVJIkSZIkaTYYuG2Gatl5582RsRpDXVa4LV4wOGl7wWB3a8D1Q+OU0sGBoNbBlNKR6tSptaPVGoOVIMLATZIkSZIkzQ0Dt81Qc9OE6YxWk6EuK9yazUfDgQ1jjWu4VTpaw23j6NTAbaxa2+T3L0mSJEmS1A2TiM1QVxVufQic5jqvGqvWuPGu9eNruA0E/Pb2+6acN9pU0da6wi1dv02SJEmSJM0pA7fNUItcqa1emiY02m7hILst2abn63tx5U2rgWLtNoA7146wbmRqB9IVK++etL1xbOo5YzUr3CRJkiRJ0twyidgM1ZsmnHvMwbz50H0A2nbxHBmrMbwJFV4/P+HZLFowyO//7Qiuf/8RHPaIB/LI3bbv+X6dWF+Ga3++/+4APOVhSxlo0fTgvg2jk7ZbTylNGyZIkiRJkqQ5ZeC2GapPKR2sBIsXFOucbRidWt0FRYXbplR41YOuiKBSCYYHKy2nbvZT/b3Uu6MOVoJsMY22uept41jrKaVWuEmSJEmSpLk0OPMpnYmI/YDDgXXAVzNzdb/urcnqTRMGKsE2ZWOBdSNVFi2Y+uccqSbbDvcvcBoeqDDSItjqp3pwVu+OWomglpCZk7qNrh0Zm3Rdq3GNVmuu4SZJkiRJkuZU10lMRJwQEbdGxE4N+w4DrgBOAf4DuDwiHtC/YapRvcKtUgm2WzgEwNU339Py3JGx3tZwizYZ1fDgXARukyvcLv79nQD87o9rJp23bmNzhZtruEmSJEmSpPnXSxJxOPDrzLyrYd+/AQmcCPwnsDfwlk0fnlqpz+gcrASPfFCxntraje2nlA73EDhd9o5ncdFxh0zZPxdTSscr3Moupf93ffFVu/j3d006b3059fS01x7I4Y/ete2UUtdwkyRJkiRJc6mXwG0Z8Kv6RkTsDjwB+I/MfG9mHg2cB/x5X0aoKapl04RKTEwpbRU2QX0Nt+4Dp50WDbfsTjo8MDD7FW5lkLawnFJa17yK24bRKoOV4Bn7PpAHLB5uOa6xTVzDTpIkSZIkqVu9JBE7Ao2lRk+hyELObth3GbDnJoxL06h3JB2sxPi0y1bTKQFGx/obOM3NlNLJFW7jmhonrB+tsrAMHBcMDrQMHcdq6RpukiRJkiRpTvWSxKwCdm/YfiYwClzcsG+4x3urA2O1iTXc6mFauxBspNrbGm7t1KeUtuoa2i/14Kw+FfZBSxYCsM3w5KYQG0ZrDYFbpWXoOFqtMVTxqyhJkiRJkuZOL0nEz4E/i4hHR8TDgJcDP83M9Q3nLANu7cP41EK9acJAJaiU65PVq96ajfS5wq1eUTeb67htHCumig6W4z7lpY8F4AGLhyefN1plYVkFNzxYYbSa1Jo+h7GqFW6SJEmSJGlu9ZLEfAhYAlwJ/Kb8/cP1gxGxEHgGsKIP41MLjU0TBmcI3Ear2d8Ktxkq6vph42htPNgDxjuxVquT3+OGsclTSmFqEDjqGm6SJEmSJGmODc58ymSZ+b8R8XzgbynWbvtKZv6g4ZSDgJXAt/oyQk3R2DShHplW20zx7LVpQjv18G5WA7exydNg6xVqY7XJz1w/Uh1vGjG+ll3DNFMoAsd+vn9JkiRJkqSZ9FT6k5nnZOaLM/MlmfmtpmPnZeb+mXlWf4Y4vYjYJyL+JSLOi4gbI2IkIm6PiO9ExDNnuPY1EXFJRKyJiNURsbwMEzdlPBERP4qILH+6DjVn0ljhNlCvcKtODdxqtWSslgwPDEw51qt6ELbyzrV9u2ezjWPV8Yo1YLyKb6ypiq9Yw60YT73BwjW3rJ50zlitxqBruEmSJEmSpDm0JSQR7wE+AOwCfJ9ieuvPgOcB50XEm1tdFBGnAKcBuwGfAU4HHgN8NyKO3oTxHE3RSGLDJtxjWuMVbpVgIMrArUWFW3165dBg/yq86nd68X9e1Ld7Nhut5qQxD7SZNts4pbR+6JWfvXjSOa7hJkmSJEmS5tqM1VcR8fReb56ZF/R6bRfOAT6YmVc07oyIg4EfASdHxNcz89aGYwcBxwLXAQdm5t3l/pOBy4BTIuLszFzZzUAiYl/gg8ApwCuAvXp+V9OoZo6HUJVKUIkiWGo2Wp3c7bMf6s+dTSPV2qQx1yvUmt/j+pEqD1i0AGBKs4S60Vqtr+9fkiRJkiRpJp1Md1xOsVZbL/o3l7GNzDytzf7zI2I58CyKdeW+0XD4DeXr++phW3nNyoj4JPBO4LXAiZ2Oo5w6+mXg9+V1r+j8XXSnWpscfA0NVMbDtUb1ddb62TRgLhoQjDZ1Vq1XqDVXuG0cq7HNcPEVa55uOnEvK9wkSZIkSdLc6iRwezdTA7cnAc+lqBD7KXAbsCvwVOChwA+AS/o3zJ6Nlq9jTfsPKV/PaXHNDygCt0PoInAD3gHsD/xpZm6MmL2Qp1qrjU8lhSJ4WnHD3VPOGy0rwvoZks1Fhdstq9dPDtzKZ46WU2k3jlX55S338vs71nLAXjsCrSvc1mwc47Z7NzBohZskSZIkSZpDMwZumXlS43ZEPBk4DngL8MnMrDUcqwBvolhT7d19HWmXImIv4FBgHXBBw/5FwO7AmsZppg2uLV8f3sWzDgSOBz6QmSt6HnSHqrWJEKruspaBWzmldHDzCtwqEdy5ZmTKM+sVbk9+/4+5e12Rpa4frQLwxL13AmCfBy4ev+6Vn/k/AIbmYMySJEmSJEl1vSQx7wHOzcyPN4ZtAJlZy8x/B37MPAZuEbEA+AqwADipcdoosKR8XT3lwsn7d+jwWdtQTCX9JT2854g4KiJWRMSKVatWdXRNtVaj0kGINN40oY9TKiuzWLlXF8BDdl40vt28hls9bAM4ZL8HAvDYPXZgyTZDHLBsp/FjV91U/CnnYhqsJEmSJElSXS9JxBOBn89wzpXAkzu9YUSsjIjs4uf0ae41QBGAPQX4GkUDg150um7dh4CHAK/JzNGZTp7ykMxPZ+YBmXnAzjvv3NE1jU0Tmu41abu+hls/mwY0V9bNhtFqsmBw5jXcgPEupQCLhgdarmU34BpukiRJkiRpDnWyhluzoFinbToP6/Ke1wEbujj/llY7y7DtdOClwJnAX2dzCjVRwbaE1maqgGt83sHAGymq6GYKIfumuWlC3Ui1xoLBiQBqNpomND43M5mNteru3TDKnpVtpzyzvoZbo4VDjcFchbEWgZskSZIkSdJc6iVwuxB4cUQ8PzPPbj4YEX8GvAj4Uac3zMxDexhH83MHgTMowrYzgFdnZrXFs9ZGxM3A7hGxW4t13PYpX3/bwWP3pwgg3xUR72pzzmgZSu3fr1CuuWlC3cjY5MDtwuvuBGDBUP8Ct122Xzj++9cuvZFXPHHPvt0bYNm/fg+AVfdtHN9Xr6pr1Rih8f0ODQSjLc750TW3c9zhj+jrOCVJkiRJktrpJXA7nqIJwXci4vzy99uBXYCDgacD68vz5kREDFNUtL0Q+BLw2ub15ZqcB7yKotPqF5qOHd5wzkx+AXyuzbGXA4uBz1NMT72zg/t1pLnC7ciDlnHahSvHK9rq6qc8do+OlqPryCMftP3477+69d6+3bfZxob3Ml7hVm01pXQiTBwaqDA6NvXPfv0da2dhhJIkSZIkSa11Hbhl5mUR8SyKIOkZ5U9SVHoB/AZ4XWZe0acxTqtskPBN4AiK8OuoGcI2gFMpArfjI+Lb9aYKEbGMYoroRpqCuIhYCiwF7sjMOwAy81zg3DbjOowicPu7zBzr6c21Ua3VJgVu++66HTDRJKFurKz2WtDHLqWNFi3oJa/tXkQwUImWa7g1VrgNDsT4e5YkSZIkSZovPSUmmXkhsF9EHAQ8nmLds9XA5eWxuXQqRdh2B3AzcEKLdcWWZ+by+kZmXhgRHwGOAa6KiLOAYYqqtJ2AN2XmyqZ7HA2cCLwLOKnv76IL1Zxc4VYP1Jor3OpdPetdPvttrgI3KN5vqzCtMUwcGqi0bJogSZIkSZI0l7pOTCLi6cC9mfnzMlyb64Ct2d7l61LghGnOW964kZnHRsRVFEHaUUANuBw4udXadPcntdrkLqXDbQK3aq1GROsGC/2wTUOH0Ol89n+v573f+xW/eNdzWNwmpNswWuXF/9n+qzQyVuPU86/jJU/YfdL+xi6lg5Xg+lVOH5UkSZIkSfOrl9Knn1AEVPcLmfmMzIwZfk5qc+0XM/PAzFyUmdtl5sHtwrbMPGm6e7U4f1l5fl+nkwKMNTVNGC67kG5sCtxGaznecKCf3vPCRwHQaYPS937vVwCccfENbc+55pbVXHPLxJpw7abB/t/1d03abjxv9fpRdlo0POWai9++yT05JEmSJEmSOtZL4HYHRVMEzZNqDSotKtyaA7dqLWdlOukLHvsgALLL5dKC9gndQNM42916aGDyPRY0VLjtvXRRyymljZ1VJUmSJEmSZlsvacxy4KA+j0NdqNZqkyrX2k0pHavOToVbPTjrtj3BaK39+mrN48w2aV7zMm6NFW6DruEmSZIkSZLuB3oJ3N4B7BsR74mIoX4PSDP77e1rJlW41UOnX91676TzPv+z33Pfxr7PaB3vR9suFGtn9brRtscqTfNT2zVkuPj6OydtNwZuwwMVRqt2KZUkSZIkSfOrlzaTxwG/AN4OvC4irgRuY2rBU2bm6zZxfGphzcYxHrTDxDTJvZcuBmDVmo2Tzpktna7d1uwhOy9qe2ywaaroN/6+dRHlt39+y/jvT1y2E40daQcrwVhZ4XbT3et6G6QkSZIkSdIm6iVwO7Lh913Ln1YSMHCbBdVa8pjddxjf3mnRMAuHKtQa5luOjs3e1Mp6xNXtGm7NVWyTj038/t9HP4WH7rx4xvud+YY/nbQ9OFBhtPwMmqfXSpIkSZIkzZVeAre9+z4KdaVayykVYYOVydMpR2ZxLbN6VVl2uYpbc1OHRo3DHRrordHD0MBEhdtsNIuQJEmSJEnqRNeBW2beMBsDUeeqtWSgqcnA4EAw1tCUYMNoddae3+mM0rvXjnDVzavHt7++4kYesGiY5zxq10lr0AHccs9E49teA7d66FirZc/TXiVJkiRJkjZVLxVummdjTV1KoVy/rGFK6XTVZP0y05TS4799Nd+/+rbx7StvWs3ff+VyTn/dk3jqPksnnfs3X7x0/PfhFoFbxMzPGxoMRqs1HvL277PHTtsAjL9KkiRJkiTNlU2adxcRAxGxS0Ts2eqnX4PUhMykllPXQxusVManUwJsHJ3NKaXlWGY47+Lr72q5f83Gqd1KG8O04cGpX8tr33v4pO0PvOgxU84ZqlTGQ8cb7yoq5o551sNnGKUkSZIkSVJ/9VThFhGPAT4APBNY0Oa07PX+aq9aBkpTKtwGmivcZnNKabmG2wyJW7tpnbWZKtUGpl442FT11rxd7Ivxz6duukYNkiRJkiRJs6HrQCwi9gMuLDd/BLwAuBK4HXg8sBT4CfCHPo1RDeqh2sCUpgnBWHVuppROVLhNn5xFm7Br7cYxxqo1BirR8pyhFhVuU85pEcq1Wvutea07SZIkSZKk2dZLBdo7gSHgwMy8OiJqwLcy890RsQj4GHAEcGT/hqm6WllWNtA8pXSgMqlpwlU3rWa2zVTh1i7rOucXt/HWs64a3/67gx8y6XirNdyatQrXmqv+YOrnJEmSJEmSNNt6WcPtGcDZmXl1w74AyMy1wN8BdwPv2eTRaYrxCrdWTRMaKtzq66BddNwhfR9DpxlWu+mcNzd0JAX41PnXT9pu16X0Hw/bZ9pzWoZwPXY8lSRJkiRJ6lUvacRS4NqG7TFg2/pGZo5RTCl99qYNTa1Uq52t4VYtq922WzjU9zFMrOE2fYlbq8DtAYuGWTsyNu117aaBPvpBS8Z/bz2ltLN9kiRJkiRJs6mXwO0uYHHD9h1Ac0fSEWAJ6rtqfUppcxOBSoXRhi6l9V9bTbPcVONruM00pbTFt2vh0ADrR3pr6NDYvbRVmNeqmq1Vx1NJkiRJkqTZ1EsacR2wrGH7MuBZEfFAgHIdtxcCv9/k0WmKehfO5rXJhpo6dNYr3GajaUCnd6yHYo2h34KhCnesGenpuTOFZ63CxXbTUyVJkiRJkmZLL00Tfgi8LSIWlWu2nQo8D7giIi4EngDsBRzbv2Gqrj5ttDlcGmhaw22sTTDXTzMUuLHdwuLr9alXPYHBgQoLByu867u/bHnubksWcuvqDXz+yAPa3q8xcGv17E7XdZMkSZIkSZpNvQRunwF+A2wDrM3M70XEPwInAS8G1gEfpOhWqj6rtWmaMDRQYc3Y2KTzIqAyK1NK62u4TX/e6FjynEftwqGP2GV838KhyQHYdgsGGR6sMFAJXrT/7hyy3y7NtxnX2L201fpxnXYulSRJkiRJmk1dB26ZeSvwtaZ9H4uIT1I0VPhjzrSavnrWrkvpQGXylNKxWs5a2FS/a85Q4zZSrTE8ODBp38KhyduLFw5yz7pRFgxWZgwHF8w0pbRFgwTXcJMkSZIkSXOtb2lEZlYz83bDttnVbm22wUqFq25azYbRanletmws0A/12556/nXTnrdxtDqpKg2mBm7bDg+wfrTKLas3zDj9deYppS0aKVjhJkmSJEmS5ljXgVtEnBYRfx0Ru8/GgDS9dmu47bjtEABX3ngPUARus1bhVgZjG0Zr3L22fQOEkWqNBUPNgdvk7cMappveOc29YHLg9pSHLp1yfLBFW1TXcJMkSZIkSXOtlzTi1cAXgT9ExK8j4j8i4sURsVOfx6YW6tNGm6dfvuzAPQAYLRsnjNVyVjqUNluzcaztsY1jtakVbk1TTPffc4fx3/fZZfG0z1rQcG2rqaIt13BrUfUmSZIkSZI0m3ppmvAo4BDgMOBg4A3A3wEZEVcB5wE/Bi4ou5iqj6rTdCkFGCunnFbnKHC7b0P7wG1krDZl3bUFTVNKG7dnqsibaT22RQsGpuyzwk2SJEmSJM21Xpom/Ar4FfDJKOYWPgE4tPw5CHgs8E/AKLCwf0MVtG+aUA+r6oFcNZOBFlMs+23Vmo0t92dm2TRh+iml2zQEbo1NH1pprpZrtmjB1K/z0Bx8BpIkSZIkSY02KY3IworM/CDwz8D7gFUUjSyH+jA+NamNV7hN/tNNVLiVgVt19tZwa/Saz1/Scv+qNRvJhKWLF0za39w0YWEXgdvQQLBwqMKL9m+9fODSRQum7HNKqSRJkiRJmmu9TCkFICL2ZqKy7RBgKUXQthL4HMW0UvXZ2PgabpP31wO4emg122u4nXvMwRz2kfPbHt8wUkxtba46a1zD7XfvO5zrVk3MOh6bIXCLCP7vuEPZdrj113bJtkNccvyhbLdgiEeccA5g4CZJkiRJkuZe14FbRHyGImTbiyJgux04l3Lttsxc2c8BarJqpxVutdqsBm7bLZz+qzNSrQJT111rnFI6OFBhqCEQG6vWZnzuDtsOT3v8gdtNnsXcqnOpJEmSJEnSbOqlwu11QAI/Ak7MzIv7OyRNpzrjGm5l04ScuQnBpljcYr20RhvHinFM6VLaNKW0MRCbvr6tN3Mwq1aSJEmSJGmSXsp/fkrREOHZwAUR8b8RcVJEPC0iXLdtlrUL3Orbo9WJCrfKLKZN2w5P7QjaaKQM3Jq7lDaPack2s/OVedo+S4FiGqokSZIkSdJc6qVL6dMjYhvgacBhwDOBdwAnAOsi4qcU67edl5mX9XOwmpgy2ly9Vl+rbHwNt1lumlAPsh67xw4tj49XuDUFbhtHi6mmj9xte6BYd+0Je+3IZTfcTS37V+P26VcdwB1tOqhKkiRJkiTNpp6aJmTmeuCH5Q8RsQNF8HYI8FLgWRQzBHtuyqDW6uucNTcDqE/NrAdytZzdpglQhG3tKtTaVbjVK/CeWlagAfzF/ruXgVv/xrbN8AB77LRt/24oSZIkSZLUoU0OxCJiR4qw7TCKZgoP3NR7qr3RNk0TxtdwKwO5sdrsVrgBDATU2qRkI20q3OprzDWObWB87LOxipskSZIkSdLc6qVLaX066aHlz+MoupUGcC9wNsWU0h/3b5iqq1e4DTVVuA0MTHQpXb1+lOW/WTXjOmubaqAS41NYm41UWwduSxcvmPQKE40VbCgqSZIkSZK2BL1UuN0NDFEEbBuA5ZRrtgGXZmatb6PTFGNlFdjgQJsKt1py0XV3ALBupDqrY6lEUG2z7tpImy6lLztgDxYMVfizx+4+vu8Fj30Q19xyL28+9GGzN1hJkiRJkqQ50kvgdgVwLkXAdmFmujL9HBotp2QOtelSOlbLvq6FNp2BSowHa83aTSmtVIK/2P/Bk/YND1Y44QWPnJ1BSpIkSZIkzbFeupT+6WwMRJ1pX+FWbFdr2ddun9MZqETbZ21sM6VUkiRJkiRpS7fJaUhE7BgRe/RjMJrZaJsupfWCt7FqjbvWjszJWG6+ez2X/+EerrllNbVacsbFf2DDaDGN9bo/rgFgwcDsriMnSZIkSZJ0f9NT4BYRiyPiwxFxG3AH8PuGY0+KiO9HxOP7NUhNGCvniw41dRiIiKKJQSYnfOeaORnL9XesBeB5H/spX1txI2//1tV89NzfAnDahSsBK9wkSZIkSdLWp+s0JCKWABcB/wTcAvyKooFC3dUUXUz/sh8D1GRjbSrcoJjiOTZXC7g1uXX1BgDubqquM3CTJEmSJElbm17SkOOBRwFHZubjga83HszMdcD5wKGbPrzpRcQ+EfEvEXFeRNwYESMRcXtEfCcinjnDta+JiEsiYk1ErI6I5RHx/B7HMRQRb46Ii8t7rY2I30bElyJi597eXWuj9TXcKi0Ctwhq8xS4ZZu13AZajFOSJEmSJGlL1kvg9iLgfzLzS9OccwOwe29D6sp7gA8AuwDfBz4M/Ax4HnBeRLy51UURcQpwGrAb8BngdOAxwHcj4uhuBhAROwE/Bf6dognF54FPACuAw8qx9c01t9xbf+6UYwOVoNq6aeisq1fWBcGda2xcK0mSJEmStl5ddykFHgx8Y4Zz1gBLerh3t84BPpiZVzTujIiDgR8BJ0fE1zPz1oZjBwHHAtcBB2bm3eX+k4HLgFMi4uzMXNnhGL4EPBH4h8z8z6ZxBH1oTFE3Wq1x7q9ub3u8uWvoAXvt2K9Hz+i2ckpppQJPeO+5c/ZcSZIkSZKk+5tewqD7gAfOcM7eFM0UZlVmntYctpX7zweWA8PAQU2H31C+vq8etpXXrAQ+CSwAXtvJ8yPiEIpqurOaw7bynpmZ1U7u1YnRGcrXijXcJs756lFP7tejW/r8kQeM/14vuHvgdgtn9ZmSJEmSJEn3d70EbpcCz4+I7VodjIjdgCMoplnOp9Hydaxp/yHl6zktrvlB0zkzeWX5elpE7BIRr4uI4yLitRHR9ym1MzVEqMTkKaWzvX7a0MDE12fdxmq5zzXbJEmSJEnS1q2XKaX/ThFMfT8ijmo8EBGPoFgTbSHwsU0fXm8iYi+Kpg3rgAsa9i+iWFtuTeM00wbXlq8P7/BRBzacfyawbcOx0Yh4d2a+t5uxT2emhggDlcnntFrnrZ8GKxOB29qRItesN3WQJEmSJEnaWnVd4ZaZ/wOcBDwF+AVwHEBE3FFuHwQcl5kX9m+YnYuIBcBXKKaGntQ4bZSJdeVWt7m8vn+HDh9Xn1p7MsW6dvuU174IuBt4T0QcOcN4j4qIFRGxYtWqVdM+rDpD4DZYqVBt0y10NjRWs/3vtcUM4rMuu2nOni9JkiRJknR/1NOC/pn5booKsv+mCJaqQFJ0Cj0sM0/u9F4RsTIisouf06e51wDwZYow8GvAKb28v/K9dGKgfL0CeE1m/i4zV2fmt4DXl8eOm/ZBmZ/OzAMy84Cdd9552ofVA7fDH71ry+OVSnHO4/bYgSXbDHX4FnrXKv+7+Z7147//+NiDZ30MkiRJkiRJ9ze9TCkFIDN/AvykD2O4DtjQxfm3tNpZhm2nAy+lmN7515lTyr3qFWztOqjOVAHX7G5gZ+DbLZ71PWAEeHhELMnMTu/ZVr167ekPbx3MDURQrSWLFgzwsAcu3tTHzaixQUMre+607bTHJUmSJEmStkQ9B24ziYidM3P6OZJAZh7ah2cNAmdQhG1nAK9u1R00M9dGxM3A7hGxW4t13PYpX3/b4aN/Q7F+2z0tnlWLiHuBpcA2dB7itVWvcGvXDKFSCaqZ1GpF+DbbZpq92thUQZIkSZIkaWvR90QkIpZExPspKtdmXUQMA2dRhG1fAl7VKmxrcF75+twWxw5vOmcmPy5fH91iXLtQhG1rgTs6vN+0xgO3NmHaYCWo1ZJaJnOQtxmoSZIkSZIktdBVYhIRe0XEiyLiBWWg1HhsYUQcB1wP/Gu39+5F2SDhW8ALgc8Br83M6ec5wqnl6/ERsWPDvZYBbwQ2Al9oes7SiNgvIpY23esrFNVtR0bEYxrOrwAfKjfPysyxbt5XOzNWuEUwVgZulTlI3A5ctiPPedQuLY8d9fSHzPrzJUmSJEmS7o86nlIaER8D/gGoJzkjEXFsZv5HRDwD+CLwYIrA6t+Bf+vzWFs5FTiCooLsZuCEmBo0Lc/M5fWNzLwwIj4CHANcFRFnAcPAy4GdgDdl5sqmexwNnAi8i6JDa/1ed0TEUcBXgYsj4hvAKuBg4PHA74C39uONAtRy+sBtYLzCrf05/RQRvOmQffifa25n+4WD3LthIld8+xGPmPXnS5IkSZIk3R91FLhFxGsoQqca8CuK0G1f4GMRsRb4FEXHzk8B783Mlo0NZsHe5etS4IRpzlveuJGZx0bEVRTv6SiK93U5cHJmnt3NADLz6+W6cG+nCP+2A24EPgy8LzPv7uZ+0xmbocJtsL6G2xxNKYWJaaXDgwNAXwr5JEmSJEmSNmudVrgdSdFx85mZeRFARDwd+BHFVM6bgBdk5tWzMch2MvMZm3DtFymq8jo59yQaKttaHL8QeLcJEuEAACAASURBVH6vY+lUfUppu+mi/7+9Ow+TqyoTP/590+kkgEkIBpmIYgCDigI6AjowbEYUhm0UcAZHIDKKgMIwOD+XAVmEmcEFlXEcHDeCIoJsIoq4JARQcUFBXFmEGEBFSQIJQjq9vL8/7q1QqVR1OunqWpLv53nuc7vOOfeeU3XrdlW/fZZx44pVSoeGWjOkFGB8T1HP+Bb0qJMkSZIkSeoGI51nbWfg2kqwDSAzbwG+QtHb7bhWB9s2RkPl7HSNgls9UQbcWjSkFGDLyROZtmkvJ89+fkvqkyRJkiRJ6nQj7eE2lWI+slr3lvvb6uSpyQbKiFvDRRMqPdwyaVWHsymTernjzNcAcNkPF/HL3y/j+nf8bWsqlyRJkiRJ6kAj7eE2Duivk94PkJlPNa1FaqiyaMK4YeZwG8pkcCips3jEmKsMYx0s2ylJkiRJkrQxGmnADcAoSpv1DxaXoOGQ0nHBwFCSWQwvbbVKz7vKXHOSJEmSJEkbo3UJuJ0dEYPVG+XKoLXp5eaSlU320NKiI2GjDmTjIhiqDCldlyvbJG/ecyYA207frPWVS5IkSZIkdYiRzuEGxeII68JlK5uspwyibT1tkwb5wWAmg9meIaWHvXRrDnvp1i2vV5IkSZIkqZOMKOCWmW3oL6VaKweKRRMm9da/HD3jgsEh2jakVJIkSZIkSes2pFRt1lcG3CaO76mb3xPB4NBQS1cplSRJkiRJ0uoMuHWRvv5KwK3+ZUuSex55gsGhXLViqCRJkiRJklrLgFsX6RsYBGBCg4BbbznJ29BQMs4ubpIkSZIkSW1hwK2L9A0MMS5gfINg2vZbPgOAgaF0DjdJkiRJkqQ2MeDWRfoGhpg4vqfhCqQ9ZSBuwB5ukiRJkiRJbWPArYusHBhiYoMVSuHpgNuSv6xcNfxUkiRJkiRJrWXArYv0DQw2XDABoLrj2zU/fbgFLZIkSZIkSVItA25dpK+/GFLaiPO2SZIkSZIktZ8Bty7SNzDUcIVSeHpIqSRJkiRJktrHgFsXWduQ0nH2cJMkSZIkSWo7A25dpFiltPElu/6u36/6efdtt2hFkyRJkiRJklTDgFsXKQJujedwe3jpU6t+fuf+O7SiSZIkSZIkSaphwK2L9A0MMbF3ZKuUju/x0kqSJEmSJLWDUZku0tc/8jncenucz02SJEmSJKkdDLh1kZVrGVJaHXAbP85LK0mSJEmS1A5GZbpI38AQE4bp4VY9pHTCeHu4SZIkSZIktYMBty7SNzDyIaX2cJMkSZIkSWoPozJdpK9/+CGl48dVBdycw02SJEmSJKktDLh1icxked/AsIG0WVs9Y9XPva5SKkmSJEmS1BZGZbrEwFAC8NTKwYZlTtv/Bat+ru7tJkmSJEmSpNYx4NYlBgaLgNuzN9+kYZnqBRXG28NNkiRJkiSpLYzKdIn+oSEAeocZUlrdq224cpIkSZIkSRo7Bty6RKWH23BDRXvGuUqpJEmSJElSuxmV6RIDg0UPt+GGitrDTZIkSZIkqf0MuHWJ/nLRhOECadU93CIMuEmSJEmSJLWDAbcu0T9QmcOt8SXrcWVSSZIkSZKktjPg1iUGhtY+pNSAmyRJkiRJUvuNb3cDNDL95aIJvcME1Sb19rDphB6O2n2bVjVLkiRJkiRJNQy4dYlVq5QO08Ott2ccd5y5PxOGKSNJkiRJkqSxZcCtSzw9pHT4YaMTx/e0ojmSJEmSJElqwK5QXWKgXKV0vPO0SZIkSZIkdbSuDrhFxKyIeHdEzI+IByNiZUQ8EhHXRcR+azn22Ij4UUQ8ERGPR8SCiDh4PdowJSL+PSLujIil5bl+HhHnRsSW6//sVrdqSOm4rr5kkiRJkiRJG7xuj96cC5wPbAXcAFwAfA84CJgfEafUOygiPgzMBWYAnwYuBXYCro+Id4y08oiYCvwY+A+gvzzn54CVwBnATyNiq/V4XmsYrPRwW8uQUkmSJEmSJLVXt8/hdiPwgcy8ozoxIvYBvg18KCKuzMw/VOXtAbwT+C2wW2YuLdM/BPwE+HBEfC0zF46g/uOBHYCLM/O4mjbMBY4F3ga8f/2e3tP6yzncehxSKkmSJEmS1NG6uodbZs6tDbaV6TcDC4AJwB412SeU+/+oBNvKYxYCnwAmAm8eYRO2K/fX18n7arlvyrDSwUHncJMkSZIkSeoGXR1wW4v+cj9Qk/6qcn9jnWO+UVNmbX5Z7g+qk1eZD+47IzzXsH71h2WAc7hJkiRJkiR1um4fUlpXRDwPmA08CdxSlb4ZsDXwRPUw0yr3lvsdRljVZ4CjgH+OiJ2A7wIB7AXsCJyemdet15OokUUHN7Z/1mbNOJ0kSZIkSZLGyAYXcIuIicAXKYaGvqt62Cgwtdw/3uDwSvrmI6krM1dExKuACynmatu9Kvsq4Csjbffa9A0MMn5cMHF8T7NOKUmSJEmSpDHQ9vGJEbEwInIdtkuHOVcP8AVgT+AK4MPr2awcYdufCXwT+HvgH4FnAtPLn/cCfhgRuzc+A0TE8RFxe0Tc/uc//7lhuRX9Q0zqNdgmSZIkSZLU6Tqhh9tvgRXrUP739RLLYNulwJHAl4E3ZWZt4KzSg20q9a2tB1ytC4B9gMMy86tV6VdExAqKHm4fBPZtdILM/BTwKYBdd921YaCvb2CQSb1tj49KkiRJkiRpLdoecMvM2aM9R0SMBy6jCLZdBhyTmYN16vpLRDwMbB0RM+rM4zar3N8zwqorCyPcVCevkvbyEZ5rWCv6hxxOKkmSJEmS1AW6vstUREygmC/tSODzwNH1gm1V5pf7A+rkHVhTZm0mlvst6+RV0laO8FzD6hsYZKI93CRJkiRJkjpeV0dwygUSrgUOAz4LvDkzh9Zy2CfL/ekRMa3qXDOBtwN9wMU19UyPiBdGxPSac91a7s+KiHFV5XuAc8qH80b8hIaxon+ISfZwkyRJkiRJ6nhtH1I6Sp8E/g54FHgYODMiasssyMwFlQeZ+f2I+AhwGnBXRFwFTAD+AdgCODkzF9ac4x3AWRRBtLOr0t8N7AEcA7w8Iio942YDO5bt+vdRPcPS/Y8+wdRNeptxKkmSJEmSJI2hbg+4bVvupwNnDlNuQfWDzHxnRNxFEUg7HhgCfgp8KDO/NtLKM/PnEfEyisDb/sDbKFY4fRD4H+D8zHx4pOcbzvIVA/SsGUyUJEmSJElSh+nqgFtm7juKYy8BLhlh2bNZvWdbdd4DwAnr246R6usfZLcdtxrraiRJkiRJkjRKXR1w21hkJk/0DbDFphPa3RRJkiRJkjZofX19LFmyhOXLlzM4ONyajGqHnp4eJk+ezBZbbMHEiRPXfkCbGHDrAisHhxhK2GSCiyZIkiRJkjRW+vr6WLRoEdOmTWPmzJn09vZSZ654tUlm0t/fz7Jly1i0aBHbbLNNxwbdunqV0o3F8hUDAAwNZZtbIkmSJEnShmvJkiVMmzaN6dOnM2HCBINtHSYimDBhAtOnT2fatGksWbKk3U1qyIBbF1j2VD8A0zZzSKkkSZIkSWNl+fLlTJkypd3N0AhMmTKF5cuXt7sZDRlw6wJPrizGjG85uTO7SUqSJEmStCEYHBykt7e33c3QCPT29nb0HHsG3LpAJeC22QSn3JMkSZIkaSw5jLQ7dPp1MuDW4QaHkrd94XbARRMkSZIkSZK6gQG3DvfkygGWPtnP5pv28pKtHUcuSZIkSZLU6Qy4dbihoWJ/8qtmMXG8PdwkSZIkSZI6nQG3DjeYCUBPZw9NliRJkiRJG5nM5MYbb+Tkk0/mpS99KdOmTWPSpEm84AUv4NRTT+WRRx5pdxPbxln4O9zgUBlwG2fETZIkSZIkdY6+vj4OPPBAJkyYwN57782rX/1qBgcHmT9/PhdeeCGXX345t956K7NmzWp3U1vOgFuHGyp7uI0z4CZJkiRJkjpIT08P5513HieddBLTpk1blT40NMRJJ53E//3f/3Haaadx/fXXt7GV7eGQ0g63qodbhy93K0mSJEmSNgxz587l8MMPZ7vttmOTTTZhypQp7Lnnnlx66aWrlevt7eX0009fLdgGMG7cOM4880wAFixYULeO3/zmNxx33HHMnDmTiRMn8qxnPYu99tqLiy66aEyeU6vZw63DVQJu9nCTJEmSJEmtcOKJJ7Ljjjuy9957M2PGDBYvXswNN9zA0Ucfzd13382555671nNMmDABgPHj1ww9ff3rX+fII4+kr6+PAw44gKOOOorHHnuMn/3sZ3zwgx/kxBNPbPpzajUDbh1uKO3hJkmSJEmSWucXv/gF22+//WppK1eu5MADD+T888/nhBNOYOuttx72HJ/97GcBOOCAA1ZLf/TRR3njG9/IwMAA8+fPZ5999lkt/6GHHmrCM2g/A24dzkUTJEmSJElqv3Ou/yW/+v2ydjdjWDs+ewpnHfLiUZ+nNtgGRY+1t7/97cyfP5958+ZxzDHHNDz+xz/+Meeccw6TJ0/mvPPOWy3vkksuYdmyZZxyyilrBNsAnvOc54y6/Z3AgFuHc9EESZIkSZLUSosWLeIDH/gA8+bNY9GiRTz11FOr5T/88MMNj73nnns45JBD6O/v5/LLL18jePeDH/wAgAMPPLD5De8gBtw63OBQsXdIqSRJkiRJ7dOMnmPd4P7772f33Xdn6dKl7LXXXrzmNa9h6tSp9PT0sHDhQi655BL6+vrqHnvvvfey3377sWTJEi6//HIOPfTQNco89thjAGsdktrtDLh1uKeHlLa5IZIkSZIkaYP3kY98hMWLF3PxxRczZ86c1fK+9KUvcckll9Q97te//jWzZ89m8eLFXHnllRx22GF1y22++eZA0Utup512amrbO4lhnA63akipPdwkSZIkSdIYu++++wA4/PDD18i7+eab6x7z85//nH333ZclS5Zw9dVXNwy2Abzyla8E4Bvf+EYTWtu5DLh1OBdNkCRJkiRJrTJz5kwAFixYsFr6N7/5TT7zmc+sUf7OO+9kv/32Y/ny5Vx33XUcfPDBw57/2GOPZcqUKVx00UXccssta+S7SqlaYtBFEyRJkiRJUoucdNJJXHzxxRx55JEcfvjhbL311vziF7/gxhtv5A1veANXXHHFqrJLly5l9uzZLFmyhNmzZ3Pbbbdx2223rXHOU089ddVQ0unTp3PZZZdxxBFHsN9++3HggQey8847s2zZMu666y4efPBBHnjggZY937FiwK3DDVV6uDmkVJIkSZIkjbGdd96Zm266iTPOOIMbbriBgYEBdtllF6655ho233zz1QJujz/+OEuWLAFg3rx5zJs3r+4558yZsyrgBnDQQQdx++23r1oJ9Vvf+hbTpk3jhS98Ie9973vH9gm2iAG3DvaT3y3lXVfdBTikVJIkSZIktcYee+zB/Pnz6+ZlORIPiuGn1Y/XxYtf/GI+//nPr9ex3cA53DrYDx9YzP2P/oXXv2xrXrL11HY3R5IkSZIkSSNgD7cO9mTfIOMCLnjDLoRDSiVJkiRJkrqCPdw62BN9A2w2cbzBNkmSJEmSpC5iwK0DZSafuOk+5n5/IZtNsBOiJEmSJElSNzHg1oF+9YdlfOibdwOww19NbnNrJEmSJEmStC4MuHWgZU8NrPr59L97URtbIkmSJEmSpHVlwK0DDQwNrfp58iSHlEqSJEmSJHUTozkdZNmKfr79q0e488Glq9Im9fa0sUWSJEmSJG1cMtPFC7tAZra7CcMy4NZBfrf4Sd76+dtXS9t0ggE3SZIkSZJaoaenh/7+fiZMmNDupmgt+vv76enp3JiJAbcO8vxnPYMrTv5bAIYy2XLyRHu4SZIkSZLUIpMnT2bZsmVMnz693U3RWixbtozJkzt3oUkDbh1kk94eXrL11HY3Q5IkSZKkjdIWW2zBokWLAJgyZQq9vb0OL+0gmUl/fz/Lli1j6dKlbLPNNu1uUkMG3CRJkiRJkoCJEyeyzTbbsGTJEhYuXMjg4GC7m6QaPT09TJ48mW222YaJEye2uzkNGXCTJEmSJEkqTZw4kRkzZjBjxox2N0VdbFy7GyBJkiRJkiRtSAy4SZIkSZIkSU3U1QG3iJgVEe+OiPkR8WBErIyIRyLiuojYr8Exz4mI0yPiyoi4LyKGIiIj4vmjaMexEfGjiHgiIh6PiAURcfD6PzNJkiRJkiR1q64OuAHnAucDWwE3ABcA3wMOAuZHxCl1jtkVOA84HAjg8dE0ICI+DMwFZgCfBi4FdgKuj4h3jObckiRJkiRJ6j6Rme1uw3qLiDnAzzLzjpr0fYBvAwnMzMw/VOU9B9i2PG5ZRCwA9gFmZeZ961j/HhQBvt8Cu2Xm0jJ9JvATYDPghZm5cCTn23XXXfP2229flyZIkiRJkiRpGBHxk8zctZV1dnUPt8ycWxtsK9NvBhYAE4A9avIeysxbM3NZE5pwQrn/j0qwraxjIfAJYCLw5ibUI0mSJEmSpC7R1QG3tegv9wNjWMeryv2NdfK+UVNGkiRJkiRJG4ENMuAWEc8DZgNPAreMUR2bAVsDT1QPWa1yb7nfYSzqlyRJkiRJUmca3+4GNFtETAS+SDGc813VQz2bbGq5b7ToQiV98+FOEhHHA8eXD5+IiLub0Da11nTg0XY3QuvFaye1lvec1Frec1Jrec9JrbUu99zzxrIh9bQ94BYRC1m3J/7FzHxTg3P1AF8A9gSuAD486gaO3rCrUmTmp4BPtagtGgMRcXurJ19Uc3jtpNbynpNay3tOai3vOam1Ov2ea3vAjWKFzxXrUP739RLLYNulwJHAl4E35dguwVrpwTa1Qf7aesBJkiRJkiRpA9T2gFtmzh7tOSJiPHAZRbDtMuCYzBwc7XmHk5l/iYiHga0jYkadedxmlft7xrIdkiRJkiRJ6ixdv2hCREwArqIItn0eOHqsg21V5pf7A+rkHVhTRhsuhwR3L6+d1Frec1Jrec9JreU9J7VWR99zMbajLsdWuUDCNcDfAZ8Fjs/MoXU8xwJgH2BWZt7XoMwMiiGif8jMx6vS9wC+RzEsdrfKAg0RMRP4CbAZ8MLMXLgubZIkSZIkSVL36vaA28XAHIpVKf6X+gsULMjMBTXHza16eACwFUXgbnmZ9pnM/G5N+WOBN2dm9bFExAXAacBDFD3tJgD/ADwTODkz/2d9npskSZIkSZK6U9vncBulbcv9dODMYcotqHl8bJ0yr68p/906ZdaQme+MiLuAdwDHA0PAT4EPZebXRnIOSZIkSZIkbTi6eg63zNw3M2Mt29l1jlvbMXNrys+pl16Vf0lm7paZm2Xm5Mzcx2Bba0XEERHx8Yi4NSKWRURGxKXtbletiNgxIr4cEX+KiBURcXdEnBMRmwxzTETEsRGxICKWRMRTEfFAeZ4dWtn+ZoqIZ0bEWyLi2oi4r3xej0fEdyPinyOiY34/ed20IYuIo8vfmRkRb2l3eyoiYo+IuKG8f56MiLsi4tRyVfJGx/RGxCkR8cPy98lfIuKeiPh8RGzZyvZL1SJir4i4OiL+EBF95f5bEfF37W4b+DmnDUtEHFTeXw+V78v7I+LKiPibdretwntOG4oN8e/wiJhb9d240TZvJPV2ew83qeIMYBfgCYrhvS9sb3PWFBGvoFhEo5di+PGDwKsoemfOjojZmdlXc8wk4ErgYOBuilV4lwPPBvYCdqB7V8I9ErgI+ANwE7CIYnj364HPAAdGxJHZ5nHvXjdtyCLiucDHKX53PqPNzVklIg4DrgZWAFcAS4BDgI8Ce1L8/qg9ZgvgG8DuFD3NPwesBJ4LvJri98ufW9B8aTURcQZwLsUUKF+j+NybDrwM2Be4oW2Nw885bVgi4gPAu4DFwFco7rvnA4cBh0fEMZnZ1mCA95w2MBvi3+FfARY2ON3RwHYU3znXLjPd3Lp+A/YDZgFB8eU1gUvHuM5KPXNGULYH+FVZ/tCq9HHlTZ/Ae+oc94ky7z+BcXXye9v92o/i9XsVxR/Q42rS/4oi+JbA4V43N7ex2crfl9+hWPjnQ+V79i1jVNec8vz7jqDsFOBPQB+wa1X6JOD75Xn+sc5xXyvzTmzwXHva/Zq7bXwbRXA4gW8Dk+vkN/3zwM85t411K79DDgJ/BJ5Vk7df+Z69fwzq9Z5z22g3NtC/wxuca3PgyfI76vSRHNMxQ7ak0cjMmzLz3izvhJGIiKMi4qaIWFp2K/11RJwRxeq3zbYP8CLglsz8alW7hyj+CwdwQkREVfu2B04AfgycnnVW4M3M/jFoa0tk5vzMvL72eWXmH4FPlg/3rT3O6yY1zSkUge83A39pVCgixkfESRHxg3KowJMRcUdEvCPGZuj3EcCWwOWZeXslMTNXUPwXFeDEmja+CjgIuCozL6o9YRYGx6CtUkPl/fEBii/nb8zM5bVl6n0e+DknrbfnUfwR/cPM/FN1RmbeRNEjbI3pBbznpPW3If4dPoyjgU2AazLz0ZFU7pBSbZQi4rPAcRTdXq8BHgNeSTHkY3ZE7J+ZA02s8lXl/sbajMy8PyLuoegKvh1FbxOAoyi+NFwCTImIQyiGRi0G5mfmfU1sX6epfGlY7Rp43aTmiIgXAecDF2bmLWXAql65XuB64LU8PYRlBcV/Mz8OvILiy0czNbzvgFsoghd7RMTEfLr7/xvL/dyI2IpiyM2zKHo5fCszH25yG6WR2INiga+rgKURcRDwEop76EeZeVvtAX7OSaNyL8VUArtHxPTqP4gjYm9gMsVQMarSveekFuqSe66Rt5b7T420cgNu2uhExByKm/xa4J8y86mqvLOBs4C3Axc2sdoXlPtGcyvcS3Gj78DTN/pu5X5qmfbMqvIZERcBp2xovTYiYjxwTPnwxqr0OXjdpFEr77EvUAzd/ve1FD+dItj2P8CplfdtFAsXfAo4LiKuyszrmtjEhvddZg5ExAPAiym+GP26zKrcdzsAXwY2rTqsPyLen5nnNbGN0khU3pePUMwruFN1ZkTcAhyRmX8uH8/BzzlpvWXmkoh4N/AR4FcR8RWKoNT2wKEUQ7vfVinvPSe1Vhfdc2uIYtGVnYB7yh6zI+KQUm2M/oWi59Rx1Td56VyKD+Z/anKdU8v94w3yK+mbV6U9q9y/H7id4gafDMym+GVwEvC+5jazI5xP0QPghsz8ZlW6101qjjMpJmufU+deWqUcDvcOil5i/1r9Rb78+Z0U81500n33IYrFFmaV+a8HlgLnll/ypFaqvC9PoBiC8mqKz4OXAN8E9qaYBL3CzzlplDLzYxS/+8dT9EZ5D8Vcig8Cc2uGmnrPSa3VLfdcPceX+0+vS+X2cNNGJSI2pVhF5VHg1AZDtfsoxnlXH7eAYvx3PRdHxMU1aTdn5r7r0rRyXz32vafc/wF4XdUvpfkRcQTFf8tPi4j/zMyV61BXx4qIUyj+iP8NVcPUvG5Sc0TE7hS92i6oN5ytxg4U/0W/FzijwX33FGvedwsp5tGp56Y657kkM+espS2rVVHu6913dwDHVs0jcm1EDABfBd4LzF2HeqTRqrwvg6In28/Kx7+MiNdR/Ld9n/K/5j/Dzzlp1CLiXRQLC/w3Re/sP1KsmvhfwBcj4qWZ+S6/W0qt1WX33OoFIqYCb6AYsj53Hc5twE0bnWkUN9WWFF1WR2ousKAmbSZwLHAdcGdN3sKax5XI+VTqm1JTDopeGQA31v4HIDN/Vg6r2p7il9LP6HIRUek+/CtgdmYuqcr2ukmjVDWU9B5G9p/wypCVWQx/3z2j5vHHWPO/hC8FDqOYf2ZhTV7tfbi+992WwFfqTNr7dYovSDtExNTMbPQfTqnZKp8H91cF2wDIzKci4pvAPwO7Uwzx9nNOGoWI2JdioZJrM/O0qqyfVgW53xkRn6T4w957Tmqdbvp7rtabKKYruXykiyVUGHDTxqZyI92RmX890oMyc25tWvmhfizFH3hr5Ne4u9zv0CB/VrmvHlt+N/Aaiokk66l8+G6ylro7XkScCnwU+AVFsO1PNUW8btLoPYOn38srGvxn8dMR8WmK4PdnyrRrM/P1I62kHM6zmnI452EUw3kWrOUUdwO7lm39Sc15xlNMQj8A3F9zzA7Uue8ycygilgHTKe47A25qlcpnyEg+D/yck0bv4HK/xvxKmflkRPwIeB3FtAqVaUu856TW6KbPuVqVxRL+by11rcE53LRRycwngF8CL46ILVpY9fxyf0BtRkRsR/EL4Hes/gfkvHL/kjrHTOTpXw4Lm9bKNignt/0oxX8n9qsTbPO6Sc3RB3y2wXZHWea75ePbKIZ2Pwa8slyttFUa3ncUc15tCny/aoVSGP6+24oi2PYXimEMUqvcQhEcnhURE+rkV96vC/2ck5piYrnfskF+JX2l95zUWl12z1WXeQXFUNh7RvBP4zUYcNPG6CPABOBzEbHG5IgRMS0iRhx1H6GbKVbT2zsiDq2qaxxF13eAT9YMhfoGxY3/2ojYv+Z876PoFntzZv6xyW1tmYh4H8UiCT+h6Nk23B/DXjdpFDLzqcx8S72NYo4zKOZTe0tmXlEuyf5xYAbw3xGxxn/OI2JGROzY5KZeRREY+8eI2LWqrklAZaXRi2qO+SJFcHBOROxUdcw44IOV8zZ5mXlpWOVn2hUUv/fPrM4rPx9eS/Ef/8qK3H7OSaNza7k/PiK2rs6IiAOBPYEVwPfLZO85qbW65Z6rVlks4VPrU7lDSrVBiIi/B/6+fPhX5f5vImJu+fOjmflvAJn5uYh4OcWKPr8t51BZBGxBMVRpb+BiilXFmiIzByPizRQR9qsi4qqyztkUQ6e+R9HLq/qYlRFxLPAt4BsRcS1F9H23so1/5ulfAF2nfG7vBwYpviCdUmeI28JKN2Gvm9QW51L8V+8E4JCImA88TLFi2iyKP15Op5h7sSkyc1lEvJUi8LYgIi4HlgCHUiztfhVFEKP6mEcj4njgcuCHEXE1xb22D/DXwH3A/2tWv3Qr0QAAC1xJREFUG6V1cBrwCuD0iNgb+BHFoiKvo/j8e2tmPgZ+zklNcBXwHYoVgX9dvjf/SDG32cEU80e9JzMXg/ec1Awb4t/hVc9tCvAPFHMBX7K+DXBz6/oNOJtiZZFG28I6xxwMfA34U3kT/ZHii/B5wAtHUOe+5bnnrEM7dwSupOi90UcxVvwcYJO1HHNFVTsfpBg//px2v+5jfM0SWOB1c3Mb+63qfnxLnbygWDV4HkXgayVF0O27FCuePncE559Tnn/fdWjTnsANFHPTPAX8HPhXoGeYY/Yofz8sLtv5W+DDwLR2v8ZuG+9G8YfER4AHyvflYoqJnl/ZoLyfc25u67kBvcCpwA+AZRTDuv9U3lOvaXCM95yb23pubNh/h59Y1vOl9X19ojyRJEmSJEmSpCZwDjdJkiRJkiSpiQy4SZIkSZIkSU1kwE2SJEmSJElqIgNukiRJkiRJUhMZcJMkSZIkSZKayICbJEmSJEmS1EQG3CRJkiRJkqQmMuAmSZLUBBGxICKy3e1opoiYFRHXRsQfIyIj4rExqGODe90kSZLGt7sBkiRJFVWBl0XACzJzRZ0yC4HnAb2ZOdDC5m1UIqIH+ArwfOALwEPAGtejqvy6Bs3enJlz17uBHaJ8P5KZM9vbEkmS1EkMuEmSpE60DXAqcH67G7IR2xbYEfh0Zh4/gvLn1Ek7FZgKXAjU9o67s9wfA2y6vo2UJEnqRAbcJElSp1kKJPDeiPhMZj7a7gZtpJ5d7n8/ksKZeXZtWkTMoQi4fSwzFzY4btH6NU+SJKlzOYebJEnqNE8C5wJTgLNGckBE7FvOMXZ2g/yFlaF/VWlzymPmRMT+EXFrRDwREX+OiIsjYvOy3Msi4msRsbTM/2pEzBymLRMj4ryIeCAi+iLitxFxVkRMaFD+hRExNyIeLMs/EhGXRcQL6pSdW7Z5u4g4OSLuioinImLBCF+nl0fE1RHxp7Ku30XE/0bEjJpyCdxcPjyrrLPh6zsa9eZwq76eEbFrRNwYEY+X1+DqiHhuWW67iLi8vGZPRcRNEbFLg3o2jYj3RsSdEfGX8lreFhFH1SkbEXFsRHy/PPeK8vp8MyL+obqNFMObn1f1GmVEzK053/pe49Mi4jdl/Q9FxEcjYkqdY3aOiC+V7/O+ss0/jYiPRUTvOlwOSZLUJPZwkyRJnegTwDuAt0XExzPznjGs61DgYOBrwCeBPYA5wLYR8R5gHnAr8FlgJ+AQYPuI2Ckzh+qc78vAbsBVQD9wGHA2sGtEHJqZq4JLEXEAcA3QC1wP3Ac8B3g9cFBE7JeZP61Tx4XAXsDXgRuAwbU9yYg4GLgaiLJtvwNeDpwIHBYRe1b1QjsHmAkcSxF4W1CmL6C1dgPeXbbh0xSv/+uBnSLiUOC7wG+Az1MEvl4PfDsitsvMJyonKYOn84GXAT8FPkfxj+fXApdFxIsz84yqev8DeC/wAMX1fByYUbbnSOAKYCHF63RqeczHqo6vDJcdzTX+KLB3Wf91ZVtPBfaKiL+tzG8YETsDP6ToFfrVss1TKObeOwk4g+J9KEmSWikz3dzc3Nzc3Nw6YqMIGjxU/nxE+fiamjILy/TxVWn7lmlnNzjvQmBhTdqc8pgBYJ+q9HHAt8u8JcA/1Rz32TLvsJr0BWX6PcC0qvRJwG1l3tFV6dMohs8+CuxYc64XA08AP61Jn1ue52Fg23V4XZ9R1jMI7FWT9+7ynN+qSR/2NR1hvZVrNXOYMguKr6R1685hXv8lwOk1ee8r8/6lwev2rpr0ScCNwBDw0qr0xRSLRGxap73T1/beatI1fhR4Xs378uoy731V6RfUez9W1T9ufa+fm5ubm5ub2/pvDimVJEkdKTOvoghUvS4i/nYMq/pSZlaGT5JFr7UvlA9/kZlfrCn/+XL/0gbnOzczl1adbwVFbymA46rKHQNsDpyVmb+qPkFm/pKiR9fLImLHOnV8MDMfGOY51ToMeCZwRWbeWpN3AUXQaP+I2GYdztkK363z+l9S7h9nzUU11rg2EfFM4E3A7Zn5werC5bV5N0WvvzfWnKufOj0Hc93mFBzNNb4wM39XVX4I+H8UwcHj6pR/qk5bl2b9XpiSJGmMOaRUkiR1sncC3wcuiIhXZmau7YD1cHudtMpCAT+pk/dwuX9Og/PdXCftVoqedC+rSvubcr9Lg7nRdij3LwJ+VZP3owZ1N/LX5X5+bUZmDkTELRRDSF8GdNIiBsNdmzszszYgVu/a7Ab0AI3moKvMcfaiqrQvAicDv4yIKymu6W2Z+fg6tB1Gd43XeB9l5v0R8SAwMyI2z8zHKIa3/gvwlYi4CvgO8L3M/O06tlWSJDWRATdJktSxMvO2MohwBPAGiuBCs9ULogyMIK/RZPSP1CZk5mBELAaeVZX8zHL/1rW07xl10v64lmNqTS33f2iQX0nffB3PO9bW6dqUwUNY/dpUXufdyq2R6tf5X4HfUvQke0+5DUTEDcA7M/O+EbV+dNd4jfdR6Y8U89VNBR7LzB9FxF7A6RT3ydEAEXE3cE5mfmmEbZUkSU3kkFJJktTp3kMxvO+/osFKnxTD7KDxPxOnNkgfC1vVJkRED0XwZVlVciVgtEtmxjDbJbXno5iza11U6vqrBvkzasptSCrP6aNreZ33qxyQmYOZeWFm7kJxPQ8HrqVYYOPGiJi4jnWvzzVe431UqlzDVdcqM2/LzIMp5mzbk2KV360oFoR49QjbKkmSmsiAmyRJ6mjl0Lj/BbalGOZXT2XOtOfWZkTE82ltz6196qTtRREMvKMq7QdVeWOtUu++tRkRMR6ozJFXb7XMbvcjioDser3OmfmnzLwmM99AMSR3e+AlVUUGKYas1jOaa7zG+ygitqN4jy8sh5PWtrUvM7+fmWcCp5TJh61H3ZIkaZQMuEmSpG7wfuAximFz9Ybf/Yai99hhEbFq2GZEbAL8d0ta+LT3RcS0qjZMAv6rfHhxVbmLKZ7TWRGxe+1JImJcROzbpDZ9hWJVz6Mi4pU1eacC2wHfycxOmr+tKTLzTxRzsu0aEe8rA4yriYjtI2Lb8ueJETE7yrGpVWV6gS3Kh09WZS0Gtizfa7VGc43/JSKeV10W+BDF9/eLq9L3ioh6PTgrPeSerJMnSZLGmHO4SZKkjpeZSyLiP4EPNsjvj4gLgfcBd0TEtRTfc/anmGT/9/WOGyO/pphs/yqKobCHUfSK+jpPr35KZi6OiCMohir+ICLmAb+k6I21DcWE+88EJo22QZn5REQcB1wJ3FwuBLAIeDnwGop5wd422no62DuAWRSB26Mj4rsUc6Q9m2LBgt2Ao4AHgE0oFh5YGBE/BH5HcQ32L8t+NTN/XXXueeXxN5aLT/QBP8vM60d5jb8H3BkRV1AMH30tsAvFQh7V98E7gddExALgfuAJ4MXAgRQ9Pz+1Pi+YJEkaHQNukiSpW/w3cBLFapr1nEXRm+etwPEUQaTLgbNZcwXIsfQGisDfP1EEdB4u23B+7SqrmTkvInYG/o0ioLIXsJIiQDgfuLpZjcrM6yJiT+Dfy7qmUrxGnwTOzcxWBiVbKjOXRcQ+FO+LN1LMyTaJIuh2L8UiCd8ui/8FeDewH7AH8PfAcopFFE4EPldz+vMohiwfQjF/Wg9wCXB9Wff6XuN/BV5H8X6eSdGT7kLgzMxcUVXufykCa68o6x8PPFSmX5CZvxvRiyRJkpoqar73SZIkSWqTiJgLHAtsm5kL29saSZK0vpzDTZIkSZIkSWoiA26SJEmSJElSExlwkyRJkiRJkprIOdwkSZIkSZKkJrKHmyRJkiRJktREBtwkSZIkSZKkJjLgJkmSJEmSJDWRATdJkiRJkiSpiQy4SZIkSZIkSU1kwE2SJEmSJElqov8PTe6//RViCPYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if __name__=='__main__':\n",
    "    seed = 1\n",
    "\n",
    "    torch.manual_seed(seed)\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed)\n",
    "\n",
    "    torch.set_num_threads(1)\n",
    "\n",
    "    env_id = \"PongNoFrameskip-v4\"\n",
    "    envs = [make_env_a2c_atari(env_id, seed, i, log_dir) for i in range(config.num_agents)]\n",
    "    envs = SubprocVecEnv(envs) if config.num_agents > 1 else DummyVecEnv(envs)\n",
    "\n",
    "    obs_shape = envs.observation_space.shape\n",
    "    obs_shape = (obs_shape[0] * 4, *obs_shape[1:])\n",
    "\n",
    "    model = Model(env=envs, config=config)\n",
    "\n",
    "    current_obs = torch.zeros(config.num_agents, *obs_shape,\n",
    "                    device=config.device, dtype=torch.float)\n",
    "\n",
    "    def update_current_obs(obs):\n",
    "        shape_dim0 = envs.observation_space.shape[0]\n",
    "        obs = torch.from_numpy(obs.astype(np.float32)).to(config.device)\n",
    "        current_obs[:, :-shape_dim0] = current_obs[:, shape_dim0:]\n",
    "        current_obs[:, -shape_dim0:] = obs\n",
    "\n",
    "    obs = envs.reset()\n",
    "    update_current_obs(obs)\n",
    "\n",
    "    model.rollouts.observations[0].copy_(current_obs)\n",
    "    \n",
    "    episode_rewards = np.zeros(config.num_agents, dtype=np.float)\n",
    "    final_rewards = np.zeros(config.num_agents, dtype=np.float)\n",
    "\n",
    "    start=timer()\n",
    "\n",
    "    print_step = 1\n",
    "    print_threshold = 10\n",
    "    \n",
    "    for frame_idx in range(1, config.MAX_FRAMES+1):\n",
    "        for step in range(config.rollout):\n",
    "            with torch.no_grad():\n",
    "                values, actions, action_log_prob = model.get_action(model.rollouts.observations[step])\n",
    "            cpu_actions = actions.view(-1).cpu().numpy()\n",
    "    \n",
    "            obs, reward, done, _ = envs.step(cpu_actions)\n",
    "\n",
    "            episode_rewards += reward\n",
    "            masks = 1. - done.astype(np.float32)\n",
    "            final_rewards *= masks\n",
    "            final_rewards += (1. - masks) * episode_rewards\n",
    "            episode_rewards *= masks\n",
    "\n",
    "            rewards = torch.from_numpy(reward.astype(np.float32)).view(-1, 1).to(config.device)\n",
    "            masks = torch.from_numpy(masks).to(config.device).view(-1, 1)\n",
    "\n",
    "            current_obs *= masks.view(-1, 1, 1, 1)\n",
    "            update_current_obs(obs)\n",
    "\n",
    "            model.rollouts.insert(current_obs, actions.view(-1, 1), action_log_prob, values, rewards, masks)\n",
    "            \n",
    "        with torch.no_grad():\n",
    "            next_value = model.get_values(model.rollouts.observations[-1])\n",
    "\n",
    "        model.rollouts.compute_returns(next_value, config.GAMMA)\n",
    "            \n",
    "        value_loss, action_loss, dist_entropy = model.update(model.rollouts)\n",
    "        \n",
    "        model.rollouts.after_update()\n",
    "\n",
    "        if frame_idx % 100 == 0:\n",
    "            try:\n",
    "                clear_output()\n",
    "                end = timer()\n",
    "                total_num_steps = (frame_idx + 1) * config.num_agents * config.rollout\n",
    "                print(\"Updates {}, Num Timesteps {}, FPS {},\\nMean/Median Reward {:.1f}/{:.1f}, Min/Max Reward {:.1f}/{:.1f},\\nEntropy {:.5f}, Value Loss {:.5f}, Policy Loss {:.5f}\".\n",
    "                format(frame_idx, total_num_steps,\n",
    "                       int(total_num_steps / (end - start)),\n",
    "                       np.mean(final_rewards),\n",
    "                       np.median(final_rewards),\n",
    "                       np.min(final_rewards),\n",
    "                       np.max(final_rewards), dist_entropy,\n",
    "                       value_loss, action_loss))\n",
    "                plot(log_dir, \"PongNoFrameskip-v4\", 'a2c', \n",
    "                     config.MAX_FRAMES * config.num_agents * config.rollout)\n",
    "            except IOError:\n",
    "                pass\n",
    "\n",
    "    model.save_w()\n",
    "    envs.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}