monte-carlo-reinforce.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from collections import namedtuple, deque\n",
    "import torch.optim as optim\n",
    "import datetime\n",
    "import gymnasium as gym\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.special import softmax\n",
    "import numpy as np\n",
    "from collections import deque, namedtuple\n",
    "from torch.distributions import Categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.01369617, -0.02302133, -0.04590265, -0.04834723], dtype=float32),\n",
       " {})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = gym.make('CartPole-v1')\n",
    "env.reset(seed=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state shape 4\n",
      "no_of_actions 2\n",
      "1\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "state_shape = env.observation_space.shape[0]\n",
    "no_of_actions = env.action_space.n\n",
    "print('state shape',state_shape)\n",
    "print('no_of_actions',no_of_actions)\n",
    "print(env.action_space.sample())\n",
    "print(\"----\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class QNetwork1(nn.Module):\n",
    "\n",
    "    def __init__(self, state_size, action_size, seed,adv_type = 'avg', fc1_units=128, fc2_units=64,fc3_units = 256):\n",
    "        \"\"\"Initialize parameters and build model.\n",
    "        Params\n",
    "        ======\n",
    "            state_size (int): Dimension of each state\n",
    "            action_size (int): Dimension of each action\n",
    "            seed (int): Random seed\n",
    "            fc1_units (int): Number of nodes in first hidden layer\n",
    "            fc2_units (int): Number of nodes in second hidden layer\n",
    "        \"\"\"\n",
    "        super(QNetwork1, self).__init__()\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "        self.fc1 = nn.Linear(state_size, fc1_units)\n",
    "        self.fc2 = nn.Linear(fc1_units, fc2_units)\n",
    "        self.fc_adv = nn.Linear(fc2_units, fc3_units)\n",
    "        self.fc_value = nn.Linear(fc2_units, fc3_units)\n",
    "        self.adv = nn.Linear(fc3_units, action_size)\n",
    "        self.value = nn.Linear(fc3_units, 1)\n",
    "        self.adv_type = adv_type\n",
    "\n",
    "    def forward(self, state):\n",
    "        \"\"\"Build a network that maps state -> action values.\"\"\"\n",
    "        x = F.relu(self.fc1(state))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x_adv = F.relu(self.fc_adv(x))\n",
    "        x_adv = F.relu(self.adv(x_adv))\n",
    "        x_value = F.relu(self.fc_value(x))\n",
    "        x_value = F.relu(self.adv(x_value))\n",
    "        if self.adv_type == 'avg':\n",
    "          advAverage = torch.mean(x_adv, dim=1, keepdim=True)\n",
    "          q =  x_value + x_adv - advAverage\n",
    "        else:\n",
    "          advMax,_ = torch.max(x_adv, dim=1, keepdim=True)\n",
    "          q =  x_value + x_adv - advMax\n",
    "        return q"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Replay Buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "BUFFER_SIZE = int(1e5)  # replay buffer size\n",
    "BATCH_SIZE = 64         # minibatch size\n",
    "GAMMA = 0.99            # discount factor\n",
    "LR = 5e-4               # learning rate\n",
    "UPDATE_EVERY = 20       # how often to update the network (When Q target is present)\n",
    "\n",
    "class ReplayBuffer:\n",
    "    \"\"\"Fixed-size buffer to store experience tuples.\"\"\"\n",
    "\n",
    "    def __init__(self, action_size, buffer_size, batch_size, seed):\n",
    "        \"\"\"Initialize a ReplayBuffer object.\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "            action_size (int): dimension of each action\n",
    "            buffer_size (int): maximum size of buffer\n",
    "            batch_size (int): size of each training batch\n",
    "            seed (int): random seed\n",
    "        \"\"\"\n",
    "        self.action_size = action_size\n",
    "        self.memory = deque(maxlen=buffer_size)\n",
    "        self.batch_size = batch_size\n",
    "        self.experience = namedtuple(\"Experience\", field_names=[\"state\", \"action\", \"reward\", \"next_state\", \"done\"])\n",
    "        self.seed = random.seed(seed)\n",
    "\n",
    "    def add(self, state, action, reward, next_state, done):\n",
    "        \"\"\"Add a new experience to memory.\"\"\"\n",
    "        e = self.experience(state, action, reward, next_state, done)\n",
    "        self.memory.append(e)\n",
    "\n",
    "    def sample(self):\n",
    "        \"\"\"Randomly sample a batch of experiences from memory.\"\"\"\n",
    "        experiences = random.sample(self.memory, k=self.batch_size)\n",
    "\n",
    "        states = torch.from_numpy(np.vstack([e.state for e in experiences if e is not None])).float().to(device)\n",
    "        actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).long().to(device)\n",
    "        rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(device)\n",
    "        next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences if e is not None])).float().to(device)\n",
    "        dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)\n",
    "\n",
    "        return (states, actions, rewards, next_states, dones)\n",
    "\n",
    "    def __len__(self):\n",
    "        \"\"\"Return the current size of internal memory.\"\"\"\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TutorialAgent():\n",
    "\n",
    "    def __init__(self, state_size, action_size, seed,adv_type):\n",
    "\n",
    "        ''' Agent Environment Interaction '''\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.seed = random.seed(seed)\n",
    "\n",
    "        ''' Q-Network '''\n",
    "        self.qnetwork_local = QNetwork1(state_size, action_size, seed, adv_type).to(device)\n",
    "        self.qnetwork_target = QNetwork1(state_size, action_size, seed, adv_type).to(device)\n",
    "        self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)\n",
    "\n",
    "        ''' Replay memory '''\n",
    "        self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed)\n",
    "\n",
    "        ''' Initialize time step (for updating every UPDATE_EVERY steps)           -Needed for Q Targets '''\n",
    "        self.t_step = 0\n",
    "\n",
    "    def step(self, state, action, reward, next_state, done):\n",
    "\n",
    "        ''' Save experience in replay memory '''\n",
    "        self.memory.add(state, action, reward, next_state, done)\n",
    "\n",
    "        ''' If enough samples are available in memory, get random subset and learn '''\n",
    "        if len(self.memory) >= BATCH_SIZE:\n",
    "            experiences = self.memory.sample()\n",
    "            self.learn(experiences, GAMMA)\n",
    "\n",
    "        \"\"\" +Q TARGETS PRESENT \"\"\"\n",
    "        ''' Updating the Network every 'UPDATE_EVERY' steps taken '''\n",
    "        self.t_step = (self.t_step + 1) % UPDATE_EVERY\n",
    "        if self.t_step == 0:\n",
    "            self.qnetwork_target.load_state_dict(self.qnetwork_local.state_dict())\n",
    "\n",
    "    def act(self, state, eps=0.):\n",
    "\n",
    "        state = torch.from_numpy(state).float().unsqueeze(0).to(device)\n",
    "        self.qnetwork_local.eval()\n",
    "        with torch.no_grad():\n",
    "            action_values = self.qnetwork_local(state)\n",
    "        self.qnetwork_local.train()\n",
    "\n",
    "        ''' Epsilon-greedy action selection (Already Present) '''\n",
    "        if random.random() > eps:\n",
    "            return np.argmax(action_values.cpu().data.numpy())\n",
    "        else:\n",
    "            return random.choice(np.arange(self.action_size))\n",
    "\n",
    "    def act_softmax(self, state, tau=1.0):\n",
    "\n",
    "        state = torch.from_numpy(state).float().unsqueeze(0).to(device)\n",
    "        self.qnetwork_local.eval()\n",
    "        with torch.no_grad():\n",
    "            action_values = self.qnetwork_local(state)\n",
    "        self.qnetwork_local.train()\n",
    "\n",
    "        ''' Softmax action selection '''\n",
    "        x = [action_value/tau for action_value in action_values.cpu().data.numpy()][0]\n",
    "        return np.random.choice(np.arange(self.action_size), p=softmax(x))\n",
    "\n",
    "    def learn(self, experiences, gamma):\n",
    "        \"\"\" +E EXPERIENCE REPLAY PRESENT \"\"\"\n",
    "        states, actions, rewards, next_states, dones = experiences\n",
    "\n",
    "        ''' Get max predicted Q values (for next states) from target model'''\n",
    "        Q_targets_next = self.qnetwork_target(next_states).detach().max(1)[0].unsqueeze(1)\n",
    "\n",
    "        ''' Compute Q targets for current states '''\n",
    "        Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))\n",
    "\n",
    "        ''' Get expected Q values from local model '''\n",
    "        Q_expected = self.qnetwork_local(states).gather(1, actions)\n",
    "\n",
    "        ''' Compute loss '''\n",
    "        loss = F.mse_loss(Q_expected, Q_targets)\n",
    "\n",
    "        ''' Minimize the loss '''\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "\n",
    "        ''' Gradiant Clipping '''\n",
    "        \"\"\" +T TRUNCATION PRESENT \"\"\"\n",
    "        for param in self.qnetwork_local.parameters():\n",
    "            #print(\"param.grad. ===>\",param.grad)\n",
    "            if param.grad != None:\n",
    "              param.grad.data.clamp_(-1, 1)\n",
    "\n",
    "        self.optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Trial run to check if algorithm runs and saves the data '"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''' Defining DQN Algorithm '''\n",
    "\n",
    "state_shape = env.observation_space.shape[0]\n",
    "action_shape = env.action_space.n\n",
    "\n",
    "def dqn(n_episodes=10000, max_t=1000, eps_start=1.0, eps_end=0.01, eps_decay=0.995):\n",
    "    rewards = []\n",
    "\n",
    "    scores_window = deque(maxlen=100)\n",
    "    ''' last 100 scores for checking if the avg is more than 195 '''\n",
    "\n",
    "    eps = eps_start\n",
    "    ''' initialize epsilon '''\n",
    "\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        state,_ = env.reset()\n",
    "        score = 0\n",
    "        for t in range(max_t):\n",
    "            action = agent.act(state, eps)\n",
    "            next_state, reward, done, _, info = env.step(action)\n",
    "            agent.step(state, action, reward, next_state, done)\n",
    "            state = next_state\n",
    "            score += reward\n",
    "            if done:\n",
    "                break\n",
    "\n",
    "        scores_window.append(score)\n",
    "        rewards.append(score)\n",
    "        eps = max(eps_end, eps_decay*eps)\n",
    "        ''' decrease epsilon '''\n",
    "\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end=\"\")\n",
    "\n",
    "        if i_episode % 100 == 0:\n",
    "           print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
    "        if np.mean(scores_window)>=475.0:\n",
    "           print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
    "           break\n",
    "    return rewards\n",
    "\n",
    "''' Trial run to check if algorithm runs and saves the data '''\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Avg Advantage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tAverage Score: 39.21\n",
      "Episode 200\tAverage Score: 108.71\n",
      "Episode 300\tAverage Score: 68.014\n",
      "Episode 400\tAverage Score: 52.50\n",
      "Episode 500\tAverage Score: 94.41\n",
      "Episode 600\tAverage Score: 151.20\n",
      "Episode 700\tAverage Score: 110.41\n",
      "Episode 800\tAverage Score: 134.33\n",
      "Episode 900\tAverage Score: 138.20\n",
      "Episode 1000\tAverage Score: 59.66\n",
      "Episode 1100\tAverage Score: 182.98\n",
      "Episode 1200\tAverage Score: 184.68\n",
      "Episode 1300\tAverage Score: 205.32\n",
      "Episode 1400\tAverage Score: 95.562\n",
      "Episode 1500\tAverage Score: 83.64\n",
      "Episode 1600\tAverage Score: 57.21\n",
      "Episode 1700\tAverage Score: 101.89\n",
      "Episode 1800\tAverage Score: 98.507\n",
      "Episode 1900\tAverage Score: 161.16\n",
      "Episode 1982\tAverage Score: 479.30\n",
      "Environment solved in 1982 episodes!\tAverage Score: 479.30\n",
      "0:07:26.066295\n"
     ]
    }
   ],
   "source": [
    "begin_time = datetime.datetime.now()\n",
    "agent = TutorialAgent(state_size=state_shape,action_size = action_shape,seed = 0,adv_type=\"max\")\n",
    "dqn()\n",
    "time_taken = datetime.datetime.now() - begin_time\n",
    "print(time_taken)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "627.0\n"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
      "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
      "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "def render_policy(agent):\n",
    "    env = gym.make('CartPole-v1', render_mode='human')\n",
    "    state,_ = env.reset()\n",
    "    total_reward = 0\n",
    "    while True:\n",
    "        action = agent.act(state)\n",
    "        next_state, reward, done, _, info = env.step(action)\n",
    "        total_reward += reward\n",
    "        env.render()\n",
    "        if done:\n",
    "            break\n",
    "        state = next_state\n",
    "    print(total_reward)\n",
    "    env.close()\n",
    "# After training your agent, call the render_policy function\n",
    "render_policy(agent)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}