HER_1.0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bit Flipping with DQN + HER\n",
    "\n",
    "  This is a plain implementation of DQN + HER from @orrivlin and modification of code for better readibility from udacity drlnd\n",
    "  \n",
    "  Changes will be made further\n",
    "  * Soft Update\n",
    "  * PER\n",
    "  * DDQN\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of bits \n",
    "N = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bit Flipping Environmnet \n",
    "N : Number of bits\n",
    "S = {state vector, goal vector}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from copy import deepcopy as dc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "device  = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BfEnv:\n",
    "    def __init__(self, N):\n",
    "        self.N = N    # Number of bits\n",
    "        self.state_size = self.N*2\n",
    "        self.action_size = self.N\n",
    "        \n",
    "    def reset(self):\n",
    "        # Input : None\n",
    "        # Output : state\n",
    "        # Randomly generates a state tuple \n",
    "        state = torch.rand((1,self.N)).round()\n",
    "        goal = torch.rand((1,self.N)).round()\n",
    "        done = False\n",
    "        return torch.cat((state, goal), dim=1), done\n",
    "    \n",
    "    def step(self, s, action):\n",
    "        # Inputs: s, action\n",
    "        # s : state, action : index number of action to be taken \n",
    "        # Output : (next_state, reward, done, dist)\n",
    "        s[0, action] = 1.0 - s[0, action] #Taking action(a) and getting next_state(s_)\n",
    "        r = -1.0 # Sparse reward penalty\n",
    "        done = False\n",
    "        \n",
    "        # Calculate distance\n",
    "        dist = (s[0,0:self.N] - s[0,self.N:]).abs().sum() \n",
    "        \n",
    "        if dist == 0:\n",
    "            done = True\n",
    "            r = 0.0 # Sparse reward \n",
    "        return s, r, done, dist        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Model\n",
    "\n",
    "Defining the Network "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class QNetwork(nn.Module):\n",
    "    \"\"\" DQN (Policy) Network \"\"\"\n",
    "    def __init__(self, state_size, action_size, seed, fc1_units=128):\n",
    "        \"\"\"\n",
    "        Initialize parameters and build model\n",
    "        Parameters\n",
    "        ====\n",
    "            state_size (int): Dimension of state size\n",
    "            action_size (int): Dimenstion of action size\n",
    "            fc1_units (int): Number of nodes in first hidden layer\n",
    "            fc2_units (int): Number of nodes in second hidden layer\n",
    "            seed (int) : Random seed\n",
    "        \"\"\" \n",
    "        super(QNetwork, self).__init__()\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "        self.fc1 = nn.Linear(state_size, fc1_units)\n",
    "        self.fc2 = nn.Linear(fc1_units, action_size)\n",
    "    \n",
    "    def forward(self, state):\n",
    "        x = F.relu(self.fc1(state))\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. HER \n",
    "  The Agent sends the experiences to the HER, and HER changes the virtual goal to real goal in the experiences and adds to the replay buffer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque, namedtuple\n",
    "import numpy as np\n",
    "from copy import deepcopy\n",
    "import random\n",
    "from recordtype import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HER:\n",
    "    \"\"\" To modify experiences in a single episode \"\"\"\n",
    "    def __init__(self, N):\n",
    "        self.N = int(N)\n",
    "        self.experience = recordtype(\"Experience\", field_names=[\"s\", \"a\", \"r\", \"s_\", \"done\"])\n",
    "        self.buffer = deque()\n",
    "        \n",
    "    def reset(self):\n",
    "        self.buffer = deque()\n",
    "        \n",
    "    def add(self, state, action, reward, next_state, done):\n",
    "        \"\"\" Add a new experience to memory \"\"\"\n",
    "        e = self.experience(state, action, reward, next_state, done)\n",
    "        self.buffer.append(e)\n",
    "        \n",
    "    def update(self):\n",
    "        \"\"\"\n",
    "        Updating the virtual goals as real goals in all the experiences of an episode\n",
    "        \"\"\"\n",
    "        her_buffer = dc(self.buffer)\n",
    "        goal = her_buffer[-1].s_[0,0:self.N] # Taking s_ from last experience \n",
    "        for i in range(len(her_buffer)):\n",
    "            her_buffer[i].s[0,self.N:] = goal  # Modify s\n",
    "            her_buffer[i].s_[0,self.N:] = goal # Modify s_\n",
    "            her_buffer[i].r = -1      # Modify r\n",
    "            her_buffer[i].done = False    # Modify done\n",
    "            if ((her_buffer[i].s_[0,0:5] - goal).abs().sum() == 0): # S_(state == goal)\n",
    "                her_buffer[i].done = True\n",
    "                her_buffer[i].r = 0.0\n",
    "        return her_buffer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Replay Buffer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    \"\"\" Fixed size buffer to store experience tuple \"\"\"\n",
    "    \n",
    "    def __init__(self, buffer_size, batch_size, seed = 0):\n",
    "        \"\"\"\n",
    "        Params\n",
    "        ====\n",
    "            buffer_size(int) : maximum size of a buffer\n",
    "            batch_size(int) : size of each training batch\n",
    "            seed (int) : random seed\n",
    "        \"\"\"\n",
    "        self.memory = deque() # deque(maxlen=buffer_size)\n",
    "        self.batch_size = batch_size\n",
    "        self.experience = recordtype(\"Experience\", field_names=[\"s\", \"a\", \"r\", \"s_\", \"done\"])\n",
    "        self.seed = random.seed(seed)\n",
    "        \n",
    "    def add(self, s, a, r, s_, done):\n",
    "        \"\"\" Add a new experience to memory \"\"\"\n",
    "        e = self.experience(s, a, r, s_, done)\n",
    "        self.memory.append(e)\n",
    "    \n",
    "    def sample(self, K):\n",
    "        \"\"\"Randomly sample a batch of experiences from memory.\"\"\"\n",
    "        experiences = random.sample(self.memory, K)\n",
    "\n",
    "        states = torch.cat([e.s for e in experiences])\n",
    "        actions = torch.cat([e.a for e in experiences]).view(K,-1)\n",
    "        rewards = torch.tensor([e.r for e in experiences]).view(K, -1)\n",
    "        next_states = torch.cat([e.s_ for e in experiences])\n",
    "        dones = torch.tensor([e.done for e in experiences]).float().view(K,-1)\n",
    "  \n",
    "        return (states, actions, rewards, next_states, dones)     \n",
    "\n",
    "    def _len__(self):\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Logger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class logger:\n",
    "    \"\"\" Using logger will help you to save all the data at one place and pass is as a single variable \"\"\"\n",
    "    def __init__(self):\n",
    "        self.log = dict()\n",
    "        \n",
    "    def add_log(self,name):\n",
    "        self.log[name] = []\n",
    "        \n",
    "    def add_item(self,name,x):\n",
    "        self.log[name].append(x)\n",
    "        \n",
    "    def get_log(self,name):\n",
    "        return self.log[name]\n",
    "    \n",
    "    def get_keys(self):\n",
    "        return self.log.keys()\n",
    "    \n",
    "    def get_current(self,name):\n",
    "        return self.log[name][-1]\n",
    "    \n",
    "    def get_latest_log(self, name, latest = 100):\n",
    "        return self.log[name][(len(self.log(name))-latest):]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mean Value generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class mean_val:\n",
    "    def __init__(self):\n",
    "        self.k = 0\n",
    "        self.val = 0\n",
    "        self.mean = 0\n",
    "        \n",
    "    def append(self,x):\n",
    "        self.k += 1\n",
    "        self.val += x\n",
    "        self.mean = self.val/self.k\n",
    "        \n",
    "    def get(self):\n",
    "        return self.mean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "from collections import namedtuple, deque\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "BUFFER_SIZE = int(1e-5) # replay buffer size\n",
    "BATCH_SIZE = 64         # mini-batch size\n",
    "GAMMA = 0.99            # discount factor\n",
    "TAU = 1e-3              # for soft update of target parameters\n",
    "LR = 0.0001             # learning rate\n",
    "UPDATE_EVERY = 4        # how often update the network\n",
    "\n",
    "device  = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    " \n",
    "class Agent:\n",
    "    \"\"\" Interacts with and learns with the environment \"\"\"\n",
    "    # DQN + HER    \n",
    "    def __init__(self, state_size, action_size, seed):\n",
    "        \"\"\"\n",
    "        Params\n",
    "        ====\n",
    "            state_size (int) : dimensions of each state\n",
    "            action_size (int) : dimensions of each action\n",
    "            seed (int) : random seed\n",
    "        \"\"\"\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.seed = random.seed(seed)\n",
    "        self.N = self.state_size/2\n",
    "        self.update_target_step = 1000\n",
    "        self.step_counter = 0\n",
    "        \n",
    "        # Q-Network \n",
    "        self.qnetwork_local = QNetwork(self.state_size, self.action_size, seed).to(device)  # local model\n",
    "        self.qnetwork_target = QNetwork(self.state_size, self.action_size, seed).to(device) # target model                          # target model\n",
    "        self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)                # optimizer\n",
    "\n",
    "        # Replay Buffer - to store experiences \n",
    "        self.buffer = ReplayBuffer(BUFFER_SIZE, BATCH_SIZE, seed)\n",
    "        \n",
    "        # HER -Hindsight Experience Replay Buffer - to modify goal to a virtual goal\n",
    "        self.her = HER(self.N)\n",
    "        \n",
    "        # Initialize time step (for updating every UPDATE_EVERY steps)\n",
    "        self.update_every = 1000  # For hard update\n",
    "        #self.update_every = 4     # For Soft update\n",
    "        \n",
    "        # Epsilon # Need to define in main rather than here\n",
    "        self.epsilon = 0.1\n",
    "        self.eps_max = 0.99\n",
    "        self.eps_min = 0.05\n",
    "        self.eps = self.eps_max # start eps from eps_max\n",
    "        \n",
    "    def act(self, state, eps):\n",
    "        \"\"\" Returns action for given state as per current policy\n",
    "        Params\n",
    "        ======\n",
    "            state (array_like): current state\n",
    "            eps (float) : epsilon, for epsilon-greedy action selection\n",
    "        \"\"\"\n",
    "        self.qnetwork_local.eval()  # evaluation mode\n",
    "        with torch.no_grad():\n",
    "            Q = self.qnetwork_local(state.to(device))\n",
    "        self.qnetwork_local.train() # training mode\n",
    "        \n",
    "        rand_num = np.random.rand()\n",
    "        if (rand_num < eps):  # Exploration\n",
    "            a = torch.randint(0, Q.shape[1], (1,)).type(torch.LongTensor)\n",
    "        else:                 # Exploitation\n",
    "            a = torch.argmax(Q, dim=1)\n",
    "        return a\n",
    "    \n",
    "    def step(self, s, a, r, s_, done):\n",
    "        # Save experience in replay buffer \n",
    "        self.buffer.add(s, a, r, s_, done)\n",
    "        \n",
    "        # Save experience in her buffer\n",
    "        self.her.add(s, a, r, s_, done)\n",
    "        \n",
    "        if len(self.buffer.memory) > BATCH_SIZE:\n",
    "            # Learning every time step and update target model every update_every step  \n",
    "            loss = self.learn()\n",
    "            \n",
    "            \n",
    "    def learn(self):  \n",
    "        \"\"\"Update value parameters using given batch of experience tuples.\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "            experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples \n",
    "            gamma (float): discount factor\n",
    "        \"\"\"\n",
    "        num = len(self.buffer.memory)    # Check the length of replay buffer\n",
    "        \n",
    "        states, actions, rewards, next_states, dones = self.buffer.sample(BATCH_SIZE)\n",
    "        \n",
    "        # Get max predicted Q-values (for next_states) from target model\n",
    "        Q_targets_next = self.qnetwork_target(next_states.to(device)).detach().max(1)[0].unsqueeze(1)\n",
    "        \n",
    "        # Compute Qtargets for current states\n",
    "        Q_targets = rewards.to(device) + (GAMMA*Q_targets_next * (1-dones.to(device)))\n",
    "        \n",
    "        # Get expected Q-values from local model\n",
    "        Q_expected = self.qnetwork_local(states.to(device)).gather(1, actions.to(device))\n",
    "        \n",
    "        # learn\n",
    "        loss = F.smooth_l1_loss(Q_expected.squeeze(),Q_targets.squeeze())\n",
    "        #loss = F.mse_loss(Q_expected, Q_targets)\n",
    "        \n",
    "        # optimizer\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optimizer.step()\n",
    "        \n",
    "        # ------------------- update target network ------------------- #\n",
    "        #self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU)  \n",
    "                \n",
    "        self.step_counter += 1\n",
    "        if (self.step_counter > self.update_target_step):\n",
    "            self.hard_update(self.qnetwork_local, self.qnetwork_target)\n",
    "            self.step_counter = 0\n",
    "        \n",
    "        \n",
    "        \n",
    "    def hard_update():    \n",
    "        self.qnetwork_target.load_state_dict(self.qnetwork_local.state_dict())\n",
    "        \n",
    "    def soft_update(self, local_model, target_model, tau):\n",
    "        \"\"\"Soft update model parameters.\n",
    "        θ_target = τ*θ_local + (1 - τ)*θ_target\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "            local_model (PyTorch model): weights will be copied from\n",
    "            target_model (PyTorch model): weights will be copied to\n",
    "            tau (float): interpolation parameter \n",
    "        \"\"\"\n",
    "        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):\n",
    "            target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)\n",
    "            \n",
    "    def her_update(self):\n",
    "        her_buffer = self.her.update()\n",
    "        for e in her_buffer:\n",
    "            self.buffer.memory.append(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize env\n",
    "env = BfEnv(5)\n",
    "\n",
    "# Initialize agent\n",
    "agent = Agent(10, 5, 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dqn_her(num_episodes=5000, N=5, eps_start=1.0, eps_end=0.01, eps_decay=0.995):\n",
    "    \"\"\" Deep Q-Learning + HER\n",
    "    \n",
    "    Params\n",
    "    ======\n",
    "        num_episodes (int): maximum number of training episodes\n",
    "        max_t (int): maximum number of timesteps per episode\n",
    "        eps_start (float): starting value of epsilon, for epsilon-greedy action selection\n",
    "        eps_end (float): minimum value of epsilon\n",
    "        eps_decay (float): multiplicative factor (per episode) for decreasing epsilon\n",
    "    \"\"\"\n",
    "    log = logger()\n",
    "    scores = []                        # list containing scores from each episode\n",
    "    scores_window = deque(maxlen=100)  # last 100 scores\n",
    "    log.add_log('scores')\n",
    "    log.add_log('episodes_loss')\n",
    "    log.add_log('final_dist')\n",
    "    mean_loss = mean_val()\n",
    "    eps = eps_start                    # initialize epsilon\n",
    "    for i_episode in range(1, num_episodes+1):\n",
    "        state, _ = env.reset()\n",
    "        score = 0\n",
    "        min_dist = N\n",
    "        for t in range(N):\n",
    "            action = agent.act(state, eps)\n",
    "            next_state, reward, done, dist = env.step(state, action)\n",
    "            \n",
    "            if dist < min_dist:\n",
    "                min_dist = dist\n",
    "                \n",
    "            if (t+1) == N:  # Breaking the episode after N number of time steps\n",
    "                done = True\n",
    "                 \n",
    "            agent.step(state, action, reward, next_state, done)\n",
    "            \n",
    "            #if (loss != None): \n",
    "            #    mean_loss.append(loss) \n",
    "            state = next_state\n",
    "            score += reward\n",
    "            \n",
    "            if done:\n",
    "                break \n",
    "                \n",
    "        agent.her_update()                # her update the experience trajectory\n",
    "        scores_window.append(score)       # save most recent score\n",
    "        scores.append(score)              # save most recent score\n",
    "        log.add_item('scores', score)     # save most recent score\n",
    "        log.add_item('episodes_loss', mean_loss.get())\n",
    "        log.add_item('final_dist', min_dist)\n",
    "        eps = max(eps_end, eps_decay*eps) # decrease epsilon\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}\\tmin dist: {}'.format(i_episode, np.mean(scores_window), min_dist), end=\"\")\n",
    "        if i_episode % 100 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
    "        \n",
    "    return log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Episode 1\tAverage Score: -5.00\tmin dist: 1.0\r",
      "Episode 2\tAverage Score: -5.00\tmin dist: 1.0\r",
      "Episode 3\tAverage Score: -5.00\tmin dist: 2.0\r",
      "Episode 4\tAverage Score: -5.00\tmin dist: 2.0\r",
      "Episode 5\tAverage Score: -5.00\tmin dist: 2.0\r",
      "Episode 6\tAverage Score: -5.00\tmin dist: 1.0\r",
      "Episode 7\tAverage Score: -5.00\tmin dist: 1.0\r",
      "Episode 8\tAverage Score: -5.00\tmin dist: 2.0\r",
      "Episode 9\tAverage Score: -5.00\tmin dist: 2.0\r",
      "Episode 10\tAverage Score: -5.00\tmin dist: 3.0\r",
      "Episode 11\tAverage Score: -4.64\tmin dist: 0.0\r",
      "Episode 12\tAverage Score: -4.33\tmin dist: 0.0\r",
      "Episode 13\tAverage Score: -4.38\tmin dist: 1.0"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Expected object of backend CUDA but got backend CPU for sequence element 0 in sequence argument at position #1 'tensors'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-469c37fb41a2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlog\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdqn_her\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_episodes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mN\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps_end\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.01\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps_decay\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.995\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mnum_episodes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m5000\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-15-8379500ccf9b>\u001b[0m in \u001b[0;36mdqn_her\u001b[0;34m(num_episodes, N, eps_start, eps_end, eps_decay)\u001b[0m\n\u001b[1;32m     32\u001b[0m                 \u001b[0mdone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m             \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m             \u001b[0;31m#if (loss != None):\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-13-94339b9bc875>\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, s, a, r, s_, done)\u001b[0m\n\u001b[1;32m     75\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemory\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m             \u001b[0;31m# Learning every time step and update target model every update_every step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 77\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-13-94339b9bc875>\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     88\u001b[0m         \u001b[0mnum\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemory\u001b[0m\u001b[0;34m)\u001b[0m    \u001b[0;31m# Check the length of replay buffer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m         \u001b[0mstates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_states\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdones\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m         \u001b[0;31m# Get max predicted Q-values (for next_states) from target model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-c2c1337f19e2>\u001b[0m in \u001b[0;36msample\u001b[0;34m(self, K)\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0mstates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ms\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexperiences\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m         \u001b[0mactions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ma\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexperiences\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m         \u001b[0mrewards\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mr\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexperiences\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0mnext_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ms_\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexperiences\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Expected object of backend CUDA but got backend CPU for sequence element 0 in sequence argument at position #1 'tensors'"
     ]
    }
   ],
   "source": [
    "log = dqn_her(num_episodes=5000, N=5, eps_start=1.0, eps_end=0.01, eps_decay=0.995)\n",
    "num_episodes = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def smooth(x,window_len=11,window='hanning'):\n",
    "    if window_len<3:\n",
    "        return x\n",
    "\n",
    "    s=np.r_[x[window_len-1:0:-1],x,x[-2:-window_len-1:-1]]\n",
    "    #print(len(s))\n",
    "    if window == 'flat': #moving average\n",
    "        w=np.ones(window_len,'d')\n",
    "    else:\n",
    "        w=eval('np.'+window+'(window_len)')\n",
    "\n",
    "    y=np.convolve(w/w.sum(),s,mode='valid')\n",
    "    return y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'log' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-8f0556c12b9a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Plotting\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mY\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_log\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'scores'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mY2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msmooth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinspace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'log' is not defined"
     ]
    }
   ],
   "source": [
    "# Plotting\n",
    "\n",
    "Y = np.asarray(log.get_log('scores'))\n",
    "Y2 = smooth(Y)\n",
    "x = np.linspace(0, len(Y), len(Y))\n",
    "fig1 = plt.figure()\n",
    "ax1 = plt.axes()\n",
    "ax1.plot(x, Y, Y2)\n",
    "plt.xlabel('episodes')\n",
    "plt.ylabel('episode return')\n",
    "\n",
    "Y = np.asarray(log.get_log('episodes_loss'))\n",
    "Y2 = smooth(Y)\n",
    "x = np.linspace(0, len(Y), len(Y))\n",
    "fig2 = plt.figure()\n",
    "ax2 = plt.axes()\n",
    "ax2.plot(x, Y, Y2)\n",
    "plt.xlabel('episodes')\n",
    "plt.ylabel('average loss')\n",
    "\n",
    "Y = np.asarray(log.get_log('final_dist'))\n",
    "Y2 = smooth(Y)\n",
    "x = np.linspace(0, len(Y), len(Y))\n",
    "fig3 = plt.figure()\n",
    "ax3 = plt.axes()\n",
    "ax3.plot(x, Y, Y2)\n",
    "plt.xlabel('episodes')\n",
    "plt.ylabel('minimum distance')\n",
    "\n",
    "Y = np.asarray(log.get_log('final_dist'))\n",
    "Y[Y > 1] = 1.0\n",
    "K = 100\n",
    "Z = Y.reshape(int(num_epochs/K),K)\n",
    "T = 1 - np.mean(Z,axis=1)\n",
    "x = np.linspace(0, len(T), len(T))*K\n",
    "fig4 = plt.figure()\n",
    "ax4 = plt.axes()\n",
    "ax4.plot(x, T)\n",
    "plt.xlabel('episodes')\n",
    "plt.ylabel('sucess rate')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "buffer = ReplayBuffer(1e-5, 64, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_e():\n",
    "    s, _ = env.reset()\n",
    "    a = torch.randint(low=0, high=5, size=(1,)).type(torch.LongTensor)\n",
    "    s_, r, done, _ = env.step(s, a)\n",
    "    return s, a, r, s_, done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "for t in range(10):\n",
    "    \n",
    "    s, a, r, s_, done = get_e()\n",
    "    if (t+1 == 5):\n",
    "        done = True\n",
    "    buffer.add(s, a, r, s_, done)\n",
    "    #replay_buffer.append(([dc(s.squeeze(0).numpy()),dc(a),dc(r),dc(s_.squeeze(0).numpy()),dc(done)]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "deque([Experience(s=tensor([[0., 1., 0., 1., 0., 1., 1., 1., 1., 0.]]), a=tensor([3]), r=-1.0, s_=tensor([[0., 1., 0., 1., 0., 1., 1., 1., 1., 0.]]), done=False),\n",
       "       Experience(s=tensor([[1., 0., 1., 1., 1., 1., 0., 1., 1., 0.]]), a=tensor([3]), r=-1.0, s_=tensor([[1., 0., 1., 1., 1., 1., 0., 1., 1., 0.]]), done=False),\n",
       "       Experience(s=tensor([[1., 0., 0., 0., 1., 0., 0., 0., 1., 1.]]), a=tensor([0]), r=-1.0, s_=tensor([[1., 0., 0., 0., 1., 0., 0., 0., 1., 1.]]), done=False),\n",
       "       Experience(s=tensor([[1., 1., 0., 0., 0., 0., 0., 1., 0., 0.]]), a=tensor([3]), r=-1.0, s_=tensor([[1., 1., 0., 0., 0., 0., 0., 1., 0., 0.]]), done=False),\n",
       "       Experience(s=tensor([[0., 0., 1., 0., 1., 1., 0., 0., 1., 1.]]), a=tensor([4]), r=-1.0, s_=tensor([[0., 0., 1., 0., 1., 1., 0., 0., 1., 1.]]), done=True),\n",
       "       Experience(s=tensor([[1., 1., 1., 0., 0., 1., 0., 1., 0., 0.]]), a=tensor([2]), r=-1.0, s_=tensor([[1., 1., 1., 0., 0., 1., 0., 1., 0., 0.]]), done=False),\n",
       "       Experience(s=tensor([[1., 1., 1., 0., 1., 1., 0., 0., 1., 1.]]), a=tensor([4]), r=-1.0, s_=tensor([[1., 1., 1., 0., 1., 1., 0., 0., 1., 1.]]), done=False),\n",
       "       Experience(s=tensor([[0., 0., 0., 0., 0., 1., 1., 0., 1., 0.]]), a=tensor([3]), r=-1.0, s_=tensor([[0., 0., 0., 0., 0., 1., 1., 0., 1., 0.]]), done=False),\n",
       "       Experience(s=tensor([[0., 1., 1., 0., 1., 0., 1., 1., 1., 0.]]), a=tensor([1]), r=-1.0, s_=tensor([[0., 1., 1., 0., 1., 0., 1., 1., 1., 0.]]), done=False),\n",
       "       Experience(s=tensor([[0., 0., 0., 0., 0., 1., 0., 1., 0., 1.]]), a=tensor([1]), r=-1.0, s_=tensor([[0., 0., 0., 0., 0., 1., 0., 1., 0., 1.]]), done=False)])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "buffer.memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Experience(s=tensor([[1., 1., 1., 0., 1., 1., 0., 0., 1., 1.]]), a=tensor([4]), r=-1.0, s_=tensor([[1., 1., 1., 0., 1., 1., 0., 0., 1., 1.]]), done=False),\n",
       " Experience(s=tensor([[0., 0., 0., 0., 0., 1., 0., 1., 0., 1.]]), a=tensor([1]), r=-1.0, s_=tensor([[0., 0., 0., 0., 0., 1., 0., 1., 0., 1.]]), done=False),\n",
       " Experience(s=tensor([[0., 1., 0., 1., 0., 1., 1., 1., 1., 0.]]), a=tensor([3]), r=-1.0, s_=tensor([[0., 1., 0., 1., 0., 1., 1., 1., 1., 0.]]), done=False),\n",
       " Experience(s=tensor([[1., 0., 0., 0., 1., 0., 0., 0., 1., 1.]]), a=tensor([0]), r=-1.0, s_=tensor([[1., 0., 0., 0., 1., 0., 0., 0., 1., 1.]]), done=False),\n",
       " Experience(s=tensor([[0., 0., 1., 0., 1., 1., 0., 0., 1., 1.]]), a=tensor([4]), r=-1.0, s_=tensor([[0., 0., 1., 0., 1., 1., 0., 0., 1., 1.]]), done=True)]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es = random.sample(buffer.memory, k=5)\n",
    "es"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([4]), tensor([1]), tensor([3]), tensor([0]), tensor([4])]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[e.a for e in es]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[4.],\n",
       "        [1.],\n",
       "        [3.],\n",
       "        [0.],\n",
       "        [4.]], device='cuda:0')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.tensor([e.a for e in es]).float().view(5,-1).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[4],\n",
       "        [1],\n",
       "        [3],\n",
       "        [0],\n",
       "        [4]], device='cuda:0')"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cat([e.a for e in es]).view(5,-1).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[4],\n",
       "        [1],\n",
       "        [3],\n",
       "        [0],\n",
       "        [4]], device='cuda:0')"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cat([e.a for e in es]).view(5,-1).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp = random.sample(agent.buffer.memory, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Expected object of backend CUDA but got backend CPU for sequence element 0 in sequence argument at position #1 'tensors'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-85-b9c6c1368257>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ma\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexp\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m: Expected object of backend CUDA but got backend CPU for sequence element 0 in sequence argument at position #1 'tensors'"
     ]
    }
   ],
   "source": [
    "torch.cat([e.a for e in exp]).view(5,-1).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Expected object of backend CUDA but got backend CPU for sequence element 0 in sequence argument at position #1 'tensors'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-73-f9c679be555e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ma\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexp\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Expected object of backend CUDA but got backend CPU for sequence element 0 in sequence argument at position #1 'tensors'"
     ]
    }
   ],
   "source": [
    "a = torch.cat([e.a for e in exp]).view(5,-1)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[3],\n",
       "        [4],\n",
       "        [2],\n",
       "        [3],\n",
       "        [3]], device='cuda:0')"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = a.to(device)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[1., 0., 0., 0., 1., 0., 0., 0., 0., 1.]]), False)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = env.reset()\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'tuple' object has no attribute 'to'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-27-2913e8ccc6c3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'tuple' object has no attribute 'to'"
     ]
    }
   ],
   "source": [
    "s = s.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(buffer.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "deque([Experience(s=tensor([[0., 0., 1., 1., 0., 1., 0., 1., 0., 1.]]), a=tensor([3]), r=-1.0, s_=tensor([[0., 0., 1., 1., 0., 1., 0., 1., 0., 1.]]), done=False),\n",
       "       Experience(s=tensor([[1., 0., 0., 1., 0., 0., 0., 1., 1., 1.]]), a=tensor([2]), r=-1.0, s_=tensor([[1., 0., 0., 1., 0., 0., 0., 1., 1., 1.]]), done=False),\n",
       "       Experience(s=tensor([[0., 1., 1., 1., 1., 1., 1., 1., 1., 0.]]), a=tensor([2]), r=-1.0, s_=tensor([[0., 1., 1., 1., 1., 1., 1., 1., 1., 0.]]), done=False),\n",
       "       Experience(s=tensor([[1., 1., 1., 0., 1., 0., 0., 1., 0., 0.]]), a=tensor([3]), r=-1.0, s_=tensor([[1., 1., 1., 0., 1., 0., 0., 1., 0., 0.]]), done=False),\n",
       "       Experience(s=tensor([[1., 0., 1., 1., 0., 1., 0., 0., 0., 1.]]), a=tensor([0]), r=-1.0, s_=tensor([[1., 0., 1., 1., 0., 1., 0., 0., 0., 1.]]), done=True)])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "buffer.memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiences = random.sample(buffer.memory, k = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Experience(s=tensor([[0., 0., 1., 0., 1., 1., 1., 1., 0., 0.]]), a=tensor([1]), r=-1.0, s_=tensor([[0., 0., 1., 0., 1., 1., 1., 1., 0., 0.]]), done=False),\n",
       " Experience(s=tensor([[1., 1., 0., 0., 1., 1., 0., 0., 1., 0.]]), a=tensor([2]), r=-1.0, s_=tensor([[1., 1., 0., 0., 1., 1., 0., 0., 1., 0.]]), done=False),\n",
       " Experience(s=tensor([[1., 1., 0., 0., 1., 0., 0., 1., 0., 1.]]), a=tensor([3]), r=-1.0, s_=tensor([[1., 1., 0., 0., 1., 0., 0., 1., 0., 1.]]), done=False),\n",
       " Experience(s=tensor([[1., 1., 0., 1., 1., 0., 0., 1., 1., 1.]]), a=tensor([0]), r=-1.0, s_=tensor([[1., 1., 0., 1., 1., 0., 0., 1., 1., 1.]]), done=False),\n",
       " Experience(s=tensor([[1., 0., 0., 0., 1., 0., 1., 1., 1., 0.]]), a=tensor([1]), r=-1.0, s_=tensor([[1., 0., 0., 0., 1., 0., 1., 1., 1., 0.]]), done=False)]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experiences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "states = torch.from_numpy(np.vstack([e.s for e in experiences if e is not None])).float().to(device)\n",
    "actions = torch.from_numpy(np.vstack([e.a for e in experiences if e is not None])).long().to(device)\n",
    "rewards = torch.from_numpy(np.vstack([e.r for e in experiences if e is not None])).float().to(device)\n",
    "next_states = torch.from_numpy(np.vstack([e.s_ for e in experiences if e is not None])).float().to(device)\n",
    "dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 1., 0., 1., 1., 1., 1., 0., 0.],\n",
       "        [1., 1., 0., 0., 1., 1., 0., 0., 1., 0.],\n",
       "        [1., 1., 0., 0., 1., 0., 0., 1., 0., 1.],\n",
       "        [1., 1., 0., 1., 1., 0., 0., 1., 1., 1.],\n",
       "        [1., 0., 0., 0., 1., 0., 1., 1., 1., 0.]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1],\n",
       "        [2],\n",
       "        [3],\n",
       "        [0],\n",
       "        [1]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.],\n",
       "        [-1.],\n",
       "        [-1.],\n",
       "        [-1.],\n",
       "        [-1.]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.],\n",
       "        [0.]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Experience(s=tensor([[0., 0., 1., 0., 1., 1., 1., 1., 0., 0.]]), a=tensor([1]), r=-1.0, s_=tensor([[0., 0., 1., 0., 1., 1., 1., 1., 0., 0.]]), done=False),\n",
       " Experience(s=tensor([[1., 1., 0., 0., 1., 1., 0., 0., 1., 0.]]), a=tensor([2]), r=-1.0, s_=tensor([[1., 1., 0., 0., 1., 1., 0., 0., 1., 0.]]), done=False),\n",
       " Experience(s=tensor([[1., 1., 0., 0., 1., 0., 0., 1., 0., 1.]]), a=tensor([3]), r=-1.0, s_=tensor([[1., 1., 0., 0., 1., 0., 0., 1., 0., 1.]]), done=False),\n",
       " Experience(s=tensor([[1., 1., 0., 1., 1., 0., 0., 1., 1., 1.]]), a=tensor([0]), r=-1.0, s_=tensor([[1., 1., 0., 1., 1., 0., 0., 1., 1., 1.]]), done=False),\n",
       " Experience(s=tensor([[1., 0., 0., 0., 1., 0., 1., 1., 1., 0.]]), a=tensor([1]), r=-1.0, s_=tensor([[1., 0., 0., 0., 1., 0., 1., 1., 1., 0.]]), done=False)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experiences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "deque([Experience(s=tensor([[1., 1., 0., 0., 1., 0., 0., 1., 0., 1.]]), a=tensor([3]), r=-1.0, s_=tensor([[1., 1., 0., 0., 1., 0., 0., 1., 0., 1.]]), done=False),\n",
       "       Experience(s=tensor([[1., 1., 1., 0., 1., 0., 0., 0., 1., 0.]]), a=tensor([3]), r=-1.0, s_=tensor([[1., 1., 1., 0., 1., 0., 0., 0., 1., 0.]]), done=False),\n",
       "       Experience(s=tensor([[1., 1., 0., 1., 1., 0., 0., 1., 1., 1.]]), a=tensor([0]), r=-1.0, s_=tensor([[1., 1., 0., 1., 1., 0., 0., 1., 1., 1.]]), done=False),\n",
       "       Experience(s=tensor([[1., 1., 0., 1., 0., 1., 0., 0., 0., 0.]]), a=tensor([3]), r=-1.0, s_=tensor([[1., 1., 0., 1., 0., 1., 0., 0., 0., 0.]]), done=False),\n",
       "       Experience(s=tensor([[1., 0., 0., 0., 1., 0., 1., 1., 1., 0.]]), a=tensor([1]), r=-1.0, s_=tensor([[1., 0., 0., 0., 1., 0., 1., 1., 1., 0.]]), done=False),\n",
       "       Experience(s=tensor([[0., 0., 1., 1., 0., 1., 1., 1., 1., 0.]]), a=tensor([0]), r=-1.0, s_=tensor([[0., 0., 1., 1., 0., 1., 1., 1., 1., 0.]]), done=False),\n",
       "       Experience(s=tensor([[0., 0., 1., 0., 1., 1., 1., 1., 0., 0.]]), a=tensor([1]), r=-1.0, s_=tensor([[0., 0., 1., 0., 1., 1., 1., 1., 0., 0.]]), done=False),\n",
       "       Experience(s=tensor([[1., 1., 0., 0., 1., 0., 1., 1., 1., 0.]]), a=tensor([2]), r=-1.0, s_=tensor([[1., 1., 0., 0., 1., 0., 1., 1., 1., 0.]]), done=False),\n",
       "       Experience(s=tensor([[0., 1., 1., 1., 1., 0., 0., 0., 0., 1.]]), a=tensor([3]), r=-1.0, s_=tensor([[0., 1., 1., 1., 1., 0., 0., 0., 0., 1.]]), done=False),\n",
       "       Experience(s=tensor([[1., 1., 0., 0., 1., 1., 0., 0., 1., 0.]]), a=tensor([2]), r=-1.0, s_=tensor([[1., 1., 0., 0., 1., 1., 0., 0., 1., 0.]]), done=False)])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "buffer.memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "deque([[array([1., 1., 0., 0., 1., 0., 0., 1., 0., 1.], dtype=float32),\n",
       "        tensor([3]),\n",
       "        -1.0,\n",
       "        array([1., 1., 0., 0., 1., 0., 0., 1., 0., 1.], dtype=float32),\n",
       "        False],\n",
       "       [array([1., 1., 1., 0., 1., 0., 0., 0., 1., 0.], dtype=float32),\n",
       "        tensor([3]),\n",
       "        -1.0,\n",
       "        array([1., 1., 1., 0., 1., 0., 0., 0., 1., 0.], dtype=float32),\n",
       "        False],\n",
       "       [array([1., 1., 0., 1., 1., 0., 0., 1., 1., 1.], dtype=float32),\n",
       "        tensor([0]),\n",
       "        -1.0,\n",
       "        array([1., 1., 0., 1., 1., 0., 0., 1., 1., 1.], dtype=float32),\n",
       "        False],\n",
       "       [array([1., 1., 0., 1., 0., 1., 0., 0., 0., 0.], dtype=float32),\n",
       "        tensor([3]),\n",
       "        -1.0,\n",
       "        array([1., 1., 0., 1., 0., 1., 0., 0., 0., 0.], dtype=float32),\n",
       "        False],\n",
       "       [array([1., 0., 0., 0., 1., 0., 1., 1., 1., 0.], dtype=float32),\n",
       "        tensor([1]),\n",
       "        -1.0,\n",
       "        array([1., 0., 0., 0., 1., 0., 1., 1., 1., 0.], dtype=float32),\n",
       "        False],\n",
       "       [array([0., 0., 1., 1., 0., 1., 1., 1., 1., 0.], dtype=float32),\n",
       "        tensor([0]),\n",
       "        -1.0,\n",
       "        array([0., 0., 1., 1., 0., 1., 1., 1., 1., 0.], dtype=float32),\n",
       "        False],\n",
       "       [array([0., 0., 1., 0., 1., 1., 1., 1., 0., 0.], dtype=float32),\n",
       "        tensor([1]),\n",
       "        -1.0,\n",
       "        array([0., 0., 1., 0., 1., 1., 1., 1., 0., 0.], dtype=float32),\n",
       "        False],\n",
       "       [array([1., 1., 0., 0., 1., 0., 1., 1., 1., 0.], dtype=float32),\n",
       "        tensor([2]),\n",
       "        -1.0,\n",
       "        array([1., 1., 0., 0., 1., 0., 1., 1., 1., 0.], dtype=float32),\n",
       "        False],\n",
       "       [array([0., 1., 1., 1., 1., 0., 0., 0., 0., 1.], dtype=float32),\n",
       "        tensor([3]),\n",
       "        -1.0,\n",
       "        array([0., 1., 1., 1., 1., 0., 0., 0., 0., 1.], dtype=float32),\n",
       "        False],\n",
       "       [array([1., 1., 0., 0., 1., 1., 0., 0., 1., 0.], dtype=float32),\n",
       "        tensor([2]),\n",
       "        -1.0,\n",
       "        array([1., 1., 0., 0., 1., 1., 0., 0., 1., 0.], dtype=float32),\n",
       "        False]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "replay_buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples = random.sample(replay_buffer, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[array([1., 0., 0., 0., 1., 1., 0., 1., 0., 0.], dtype=float32),\n",
       "  tensor([3]),\n",
       "  -1.0,\n",
       "  array([1., 0., 0., 0., 1., 1., 0., 1., 0., 0.], dtype=float32),\n",
       "  False],\n",
       " [array([0., 1., 1., 0., 1., 1., 0., 1., 1., 1.], dtype=float32),\n",
       "  tensor([2]),\n",
       "  -1.0,\n",
       "  array([0., 1., 1., 0., 1., 1., 0., 1., 1., 1.], dtype=float32),\n",
       "  False],\n",
       " [array([1., 0., 1., 0., 0., 1., 1., 0., 0., 1.], dtype=float32),\n",
       "  tensor([2]),\n",
       "  -1.0,\n",
       "  array([1., 0., 1., 0., 0., 1., 1., 0., 0., 1.], dtype=float32),\n",
       "  False],\n",
       " [array([1., 0., 1., 0., 0., 1., 1., 1., 1., 1.], dtype=float32),\n",
       "  tensor([2]),\n",
       "  -1.0,\n",
       "  array([1., 0., 1., 0., 0., 1., 1., 1., 1., 1.], dtype=float32),\n",
       "  False],\n",
       " [array([1., 0., 1., 0., 0., 1., 0., 0., 1., 0.], dtype=float32),\n",
       "  tensor([3]),\n",
       "  -1.0,\n",
       "  array([1., 0., 1., 0., 0., 1., 0., 0., 1., 0.], dtype=float32),\n",
       "  False]]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "S0, A0, R1, S1, D1 = zip(*samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1., 0., 0., 0., 1., 1., 0., 1., 0., 0.], dtype=float32),\n",
       " array([0., 1., 1., 0., 1., 1., 0., 1., 1., 1.], dtype=float32),\n",
       " array([1., 0., 1., 0., 0., 1., 1., 0., 0., 1.], dtype=float32),\n",
       " array([1., 0., 1., 0., 0., 1., 1., 1., 1., 1.], dtype=float32),\n",
       " array([1., 0., 1., 0., 0., 1., 0., 0., 1., 0.], dtype=float32))"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "S0 = torch.tensor(S0, dtype=torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0., 0., 1., 1., 0., 1., 0., 0.],\n",
       "        [0., 1., 1., 0., 1., 1., 0., 1., 1., 1.],\n",
       "        [1., 0., 1., 0., 0., 1., 1., 0., 0., 1.],\n",
       "        [1., 0., 1., 0., 0., 1., 1., 1., 1., 1.],\n",
       "        [1., 0., 1., 0., 0., 1., 0., 0., 1., 0.]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([3]), tensor([2]), tensor([2]), tensor([2]), tensor([3]))"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[3],\n",
       "        [2],\n",
       "        [2],\n",
       "        [2],\n",
       "        [3]])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A0 = torch.tensor(A0, dtype=torch.long).view(5,-1)\n",
    "A0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.],\n",
       "        [-1.],\n",
       "        [-1.],\n",
       "        [-1.],\n",
       "        [-1.]])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "R1 = torch.tensor(R1, dtype=torch.float).view(5,-1)\n",
    "R1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0., 0., 1., 1., 0., 1., 0., 0.],\n",
       "        [0., 1., 1., 0., 1., 1., 0., 1., 1., 1.],\n",
       "        [1., 0., 1., 0., 0., 1., 1., 0., 0., 1.],\n",
       "        [1., 0., 1., 0., 0., 1., 1., 1., 1., 1.],\n",
       "        [1., 0., 1., 0., 0., 1., 0., 0., 1., 0.]])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S1 = torch.tensor(S1 , dtype=torch.float)\n",
    "S1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D1 = torch.tensor(D1, dtype=torch.float)\n",
    "D1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# logger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class logger:\n",
    "    def __init__(self):\n",
    "        self.log = dict()\n",
    "        \n",
    "    def add_log(self, name):\n",
    "        self.log[name] = []\n",
    "        \n",
    "    def add_item(self, name, x):\n",
    "        self.log[name].append(x)\n",
    "        \n",
    "    def get_log(self, name):\n",
    "        return self.log[name]\n",
    "    \n",
    "    def get_keys(self):\n",
    "        return self.log.keys()\n",
    "    \n",
    "    def get_current(self, name):\n",
    "        return self.log[name][-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log = logger()\n",
    "log.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "log.add_log('scores_window')\n",
    "for i in range(200):\n",
    "    log.add_item('scores_window', i/50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'scores_window': [0.0,\n",
       "  0.02,\n",
       "  0.04,\n",
       "  0.06,\n",
       "  0.08,\n",
       "  0.1,\n",
       "  0.12,\n",
       "  0.14,\n",
       "  0.16,\n",
       "  0.18,\n",
       "  0.2,\n",
       "  0.22,\n",
       "  0.24,\n",
       "  0.26,\n",
       "  0.28,\n",
       "  0.3,\n",
       "  0.32,\n",
       "  0.34,\n",
       "  0.36,\n",
       "  0.38,\n",
       "  0.4,\n",
       "  0.42,\n",
       "  0.44,\n",
       "  0.46,\n",
       "  0.48,\n",
       "  0.5,\n",
       "  0.52,\n",
       "  0.54,\n",
       "  0.56,\n",
       "  0.58,\n",
       "  0.6,\n",
       "  0.62,\n",
       "  0.64,\n",
       "  0.66,\n",
       "  0.68,\n",
       "  0.7,\n",
       "  0.72,\n",
       "  0.74,\n",
       "  0.76,\n",
       "  0.78,\n",
       "  0.8,\n",
       "  0.82,\n",
       "  0.84,\n",
       "  0.86,\n",
       "  0.88,\n",
       "  0.9,\n",
       "  0.92,\n",
       "  0.94,\n",
       "  0.96,\n",
       "  0.98,\n",
       "  1.0,\n",
       "  1.02,\n",
       "  1.04,\n",
       "  1.06,\n",
       "  1.08,\n",
       "  1.1,\n",
       "  1.12,\n",
       "  1.14,\n",
       "  1.16,\n",
       "  1.18,\n",
       "  1.2,\n",
       "  1.22,\n",
       "  1.24,\n",
       "  1.26,\n",
       "  1.28,\n",
       "  1.3,\n",
       "  1.32,\n",
       "  1.34,\n",
       "  1.36,\n",
       "  1.38,\n",
       "  1.4,\n",
       "  1.42,\n",
       "  1.44,\n",
       "  1.46,\n",
       "  1.48,\n",
       "  1.5,\n",
       "  1.52,\n",
       "  1.54,\n",
       "  1.56,\n",
       "  1.58,\n",
       "  1.6,\n",
       "  1.62,\n",
       "  1.64,\n",
       "  1.66,\n",
       "  1.68,\n",
       "  1.7,\n",
       "  1.72,\n",
       "  1.74,\n",
       "  1.76,\n",
       "  1.78,\n",
       "  1.8,\n",
       "  1.82,\n",
       "  1.84,\n",
       "  1.86,\n",
       "  1.88,\n",
       "  1.9,\n",
       "  1.92,\n",
       "  1.94,\n",
       "  1.96,\n",
       "  1.98,\n",
       "  2.0,\n",
       "  2.02,\n",
       "  2.04,\n",
       "  2.06,\n",
       "  2.08,\n",
       "  2.1,\n",
       "  2.12,\n",
       "  2.14,\n",
       "  2.16,\n",
       "  2.18,\n",
       "  2.2,\n",
       "  2.22,\n",
       "  2.24,\n",
       "  2.26,\n",
       "  2.28,\n",
       "  2.3,\n",
       "  2.32,\n",
       "  2.34,\n",
       "  2.36,\n",
       "  2.38,\n",
       "  2.4,\n",
       "  2.42,\n",
       "  2.44,\n",
       "  2.46,\n",
       "  2.48,\n",
       "  2.5,\n",
       "  2.52,\n",
       "  2.54,\n",
       "  2.56,\n",
       "  2.58,\n",
       "  2.6,\n",
       "  2.62,\n",
       "  2.64,\n",
       "  2.66,\n",
       "  2.68,\n",
       "  2.7,\n",
       "  2.72,\n",
       "  2.74,\n",
       "  2.76,\n",
       "  2.78,\n",
       "  2.8,\n",
       "  2.82,\n",
       "  2.84,\n",
       "  2.86,\n",
       "  2.88,\n",
       "  2.9,\n",
       "  2.92,\n",
       "  2.94,\n",
       "  2.96,\n",
       "  2.98,\n",
       "  3.0,\n",
       "  3.02,\n",
       "  3.04,\n",
       "  3.06,\n",
       "  3.08,\n",
       "  3.1,\n",
       "  3.12,\n",
       "  3.14,\n",
       "  3.16,\n",
       "  3.18,\n",
       "  3.2,\n",
       "  3.22,\n",
       "  3.24,\n",
       "  3.26,\n",
       "  3.28,\n",
       "  3.3,\n",
       "  3.32,\n",
       "  3.34,\n",
       "  3.36,\n",
       "  3.38,\n",
       "  3.4,\n",
       "  3.42,\n",
       "  3.44,\n",
       "  3.46,\n",
       "  3.48,\n",
       "  3.5,\n",
       "  3.52,\n",
       "  3.54,\n",
       "  3.56,\n",
       "  3.58,\n",
       "  3.6,\n",
       "  3.62,\n",
       "  3.64,\n",
       "  3.66,\n",
       "  3.68,\n",
       "  3.7,\n",
       "  3.72,\n",
       "  3.74,\n",
       "  3.76,\n",
       "  3.78,\n",
       "  3.8,\n",
       "  3.82,\n",
       "  3.84,\n",
       "  3.86,\n",
       "  3.88,\n",
       "  3.9,\n",
       "  3.92,\n",
       "  3.94,\n",
       "  3.96,\n",
       "  3.98]}"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.98"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.get_current('scores_window')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3.8, 3.82, 3.84, 3.86, 3.88, 3.9, 3.92, 3.94, 3.96, 3.98]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.get_log('scores_window')[(len(log.get_log('scores_window'))-10):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(log.get_log('scores_window'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tot_return': []}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.add_log('tot_return')\n",
    "log.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tot_return': [], 'avg_loss': []}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.add_log('avg_loss')\n",
    "log.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tot_return': [], 'avg_loss': [], 'final_dist': []}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.add_log('final_dist')\n",
    "log.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tot_return': [], 'avg_loss': [5], 'final_dist': []}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.add_item('avg_loss', 5)\n",
    "log.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tot_return': [], 'avg_loss': [5, 10, 10], 'final_dist': []}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.add_item('avg_loss', 10)\n",
    "log.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5, 10, 10]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.get_log('avg_loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.get_log('final_dist')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "class mean_log:\n",
    "    def __init__(self):\n",
    "        self.num = 0\n",
    "        self.sum = 0\n",
    "        self.mean = 0\n",
    "        \n",
    "    def append(self, x):\n",
    "        self.num += 1\n",
    "        self.sum += x\n",
    "        self.mean = self.sum / self.num\n",
    "        \n",
    "    def get(self):\n",
    "        return self.mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['tot_return', 'avg_loss', 'final_dist'])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log.get_keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}