Ex 1 HA 7.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib as plt\n",
    "import matplotlib.pyplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class four_room():\n",
    "\n",
    "\tdef __init__(self):\n",
    "\t\tself.nS = 20\n",
    "\t\tnS = self.nS\n",
    "\t\tself.nA = 4\n",
    "\n",
    "\t\tself.map = [[-1, -1, -1, -1, -1, -1, -1],\n",
    "\t\t\t\t\t[-1,  0,  1,  2,  3,  4, -1],\n",
    "\t\t\t\t\t[-1,  5,  6, -1,  7,  8, -1],\n",
    "\t\t\t\t\t[-1,  9, -1, -1, 10, -1, -1],\n",
    "\t\t\t\t\t[-1, 11, 12, 13, 14, 15, -1],\n",
    "\t\t\t\t\t[-1, 16, 17, -1, 18, 19, -1],\n",
    "\t\t\t\t\t[-1, -1, -1, -1, -1, -1, -1]]\n",
    "\t\tmap = np.array(self.map)\n",
    "\n",
    "\t\t# We build the transitions matrix P using the map.\n",
    "\t\tself.P = np.zeros((nS, 4, nS))\n",
    "\t\tfor s in range(nS):\n",
    "\t\t\ttemp = np.where(s == map)\n",
    "\t\t\tx, y = temp[0][0], temp[1][0]\n",
    "\t\t\tup = map[x-1, y]\n",
    "\t\t\tright = map[x, y+1]\n",
    "\t\t\tdown = map[x+1, y]\n",
    "\t\t\tleft = map[x, y-1]\n",
    "\n",
    "\t\t\t# Action 0: go up.\n",
    "\t\t\ta = 0\n",
    "\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\t# Up\n",
    "\t\t\tif up == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.7\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, up] += 0.7\n",
    "\t\t\t# Right\n",
    "\t\t\tif right == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, right] += 0.1\n",
    "\t\t\t# Left\n",
    "\t\t\tif left == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, left] += 0.1\n",
    "\t\t\t\n",
    "\t\t\t# Action 1: go right.\n",
    "\t\t\ta = 1\n",
    "\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\t# Up\n",
    "\t\t\tif up == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, up] += 0.1\n",
    "\t\t\t# Right\n",
    "\t\t\tif right == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.7\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, right] += 0.7\n",
    "\t\t\t# Down\n",
    "\t\t\tif down == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, down] += 0.1\n",
    "\t\t\t\n",
    "\t\t\t# Action 2: go down.\n",
    "\t\t\ta = 2\n",
    "\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\t# Right\n",
    "\t\t\tif right == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, right] += 0.1\n",
    "\t\t\t# Down\n",
    "\t\t\tif down == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.7\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, down] += 0.7\n",
    "\t\t\t# Left\n",
    "\t\t\tif left == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, left] += 0.1\n",
    "\n",
    "\t\t\t# Action 3: go left.\n",
    "\t\t\ta = 3\n",
    "\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\t# Up\n",
    "\t\t\tif up == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, up] += 0.1\n",
    "\t\t\t# Down\n",
    "\t\t\tif down == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, down] += 0.1\n",
    "\t\t\t# Left\n",
    "\t\t\tif left == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.7\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, left] += 0.7\n",
    "\t\t\t\n",
    "\t\t\t# Set to teleport back when in the rewarding state.\n",
    "\t\t\tif s == self.nS - 1:\n",
    "\t\t\t\tfor a in range(4):\n",
    "\t\t\t\t\tfor ss in range(self.nS):\n",
    "\t\t\t\t\t\tself.P[s, a, ss] = 0\n",
    "\t\t\t\t\t\tif ss == 0:\n",
    "\t\t\t\t\t\t\tself.P[s, a, ss] = 1\n",
    "\n",
    "\t\t\t\n",
    "\t\t# We build the reward matrix R.\n",
    "\t\tself.R = np.zeros((nS, 4))\n",
    "\t\tfor a in range(4):\n",
    "\t\t\tself.R[nS - 1, a] = 1\n",
    "\n",
    "\t\t# We (arbitrarily) set the initial state in the top-left corner.\n",
    "\t\tself.s = 0\n",
    "\n",
    "\t# To reset the environment in initial settings.\n",
    "\tdef reset(self):\n",
    "\t\tself.s = 0\n",
    "\t\treturn self.s\n",
    "\n",
    "\t# Perform a step in the environment for a given action. Return a couple state, reward (s_t, r_t).\n",
    "\tdef step(self, action):\n",
    "\t\tnew_s = np.random.choice(np.arange(self.nS), p=self.P[self.s, action])\n",
    "\t\treward = self.R[self.s, action]\n",
    "\t\tself.s = new_s\n",
    "\t\treturn new_s, reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = four_room()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exp_rew(env, s, a, V):\n",
    "    x = env.R[s, a] + np.sum([env.P[s, a, s_] * V[s_] for s_ in range(env.nS)])\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pi(V, env):\n",
    "    pi = np.zeros(env.nS,dtype = int)\n",
    "    for s in range(env.nS):\n",
    "        pi[s] = np.argmax([exp_rew(env, s, a, V) for a in range(env.nA)])\n",
    "    return pi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convergence(f):\n",
    "    return np.max(f) - np.min(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def VI_ar(env, epsilon=10 ** -6):\n",
    "\n",
    "    # Initialization\n",
    "    V0 = np.zeros(env.nS)\n",
    "    V1 = np.zeros(env.nS)\n",
    "    n_iter = 0\n",
    "\n",
    "    while True:\n",
    "        n_iter += 1\n",
    "        V0 = V1.copy()\n",
    "        for s in range(env.nS):\n",
    "            V1[s] = np.max([exp_rew(env, s, a, V0) for a in range(env.nA)])\n",
    "\n",
    "        if convergence(V1 - V0) < epsilon:\n",
    "            break\n",
    "\n",
    "    V = V1\n",
    "    policy = pi(V, env)\n",
    "    g = (np.max(V1 - V0) + np.min(V1 - V0)) / 2\n",
    "    b = V - np.min(V)\n",
    "\n",
    "    return V, policy, g, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([11.06132241, 11.15585494, 11.27739645, 11.38543287, 11.30271762,\n",
       "        11.15585494, 11.06132241, 11.5207192 , 11.39893747, 11.27739645,\n",
       "        11.64615254, 11.38543287, 11.5207192 , 11.64615254, 11.75418877,\n",
       "        11.8622254 , 11.30271762, 11.39893747, 11.8622254 , 11.9856963 ]),\n",
       " array([1, 1, 1, 2, 2, 2, 0, 2, 3, 2, 2, 1, 1, 1, 1, 2, 1, 0, 1, 0]),\n",
       " 0.0756256205775161,\n",
       " array([0.00000000e+00, 9.45325245e-02, 2.16074040e-01, 3.24110455e-01,\n",
       "        2.41395207e-01, 9.45325245e-02, 1.77635684e-15, 4.59396786e-01,\n",
       "        3.37615056e-01, 2.16074040e-01, 5.84830132e-01, 3.24110455e-01,\n",
       "        4.59396786e-01, 5.84830132e-01, 6.92866356e-01, 8.00902991e-01,\n",
       "        2.41395207e-01, 3.37615056e-01, 8.00902991e-01, 9.24373887e-01]))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "V, policy, gain, bias= VI_ar(env)\n",
    "V, policy, gain, bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9243738874162108"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "span_bias = convergence(bias)\n",
    "span_bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAScAAAEuCAYAAADMVdSJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAVf0lEQVR4nO3de3RV9ZnG8efNBQm3BBQRCAq1ilpmkEut1ru21FEUeplWpqLLZUVtO8u5tE4VXdOOf8xa0462M6KjVTqAFwRbp63aqhUch+UokELkXjNUBBUjIHjBS8J5548cKOhJck7OPuw3Od/PWnuRhJ1fnpNfzpO9zzn5bXN3AUA0FWkHAIBcKCcAIVFOAEKinACERDkBCIlyAhBS2HIys/PMbIOZNZnZ99LOs5eZzTazZjNbnXaW/ZnZCDNbbGZrzWyNmV2bdiZJMrPeZrbUzBqzuX6Qdqb9mVmlma0ws0fSzrI/M3vJzFaZ2UozW552nr3MrM7MHjKz9Wa2zsxOKdnXivg6JzOrlPQHSZ+XtEXSMknT3H1tqsEkmdkZkt6RNNfdx6SdZy8zGyppqLv/3sz6S2qQNDXt75mZmaS+7v6OmVVLWiLpWnd/Ls1ce5nZ30maKGmAu09OO89eZvaSpInuvi3tLPszszmS/sfd7zazXpL6uPvOUnytqEdOJ0lqcveN7v6hpPmSpqScSZLk7s9I2pF2jo9y99fc/ffZt9+WtE7S8HRTSd7mney71dktxG9EM6uXdIGku9PO0h2YWa2kMyTdI0nu/mGpikmKW07DJW3e7/0tCnBH6y7MbKSkcZKeTzmKpH2nTislNUt60t1D5JL0Y0nXScqknCMXl/SEmTWY2Yy0w2SNkvSGpJ9lT4XvNrO+pfpiUcsJXWRm/ST9XNLfuPtbaeeRJHff4+4nSqqXdJKZpX46bGaTJTW7e0PaWdpxmruPl/QXkr6VfTghbVWSxku6w93HSXpXUskeD45aTq9IGrHf+/XZj6ED2cd0fi7pPnf/Rdp5Pip7CrBY0nkpR5GkUyVdlH1sZ76kc8zs3nQj/Ym7v5L9t1nSw2p7qCNtWyRt2e/I9yG1lVVJRC2nZZKOMbNR2QfdLpb0q5QzhZZ94PkeSevc/Za08+xlZoPNrC77do3anuRYn2ooSe5+vbvXu/tItf18LXL3S1KOJUkys77ZJzWUPW2aJCn1Z4fdfaukzWY2OvuhcyWV7AmXqlINXAx3bzWzb0t6XFKlpNnuviblWJIkM3tA0lmSDjOzLZL+0d3vSTeVpLYjgemSVmUf35GkG9z9sfQiSZKGSpqTfQa2QtICdw/1tH1AQyQ93Pb7RlWS7nf336YbaZ+/lnRf9qBho6TLS/WFQr6UAACintYBKHOUE4CQKCcAIVFOAEIKX06BXh17AHIVJmouKW62cs8VvpwkhZwgkatQUXNJcbOVda7uUE4AylBJXudkZrx4CkBe3N1yfZwjJwAhUU4AQqKcAIREOQEIiXICEBLlBCAkyglASJQTgJAoJwAhUU4AQqKcAIREOQEIiXICEBLlBCAkyglASHmVk5mdZ2YbzKzJzEp2bXQA2KvTxeayV2r9g9ouI71FbZcKn+bu7V6GmMXmAOSrmMXmTpLU5O4b3f1DSfMlTUkyHAB8VD7lNFzS5v3e35L9GACUTFVSA2UvFxP1ahEAupl8yukVSSP2e78++7EDuPtdku6SeMwJQPHyOa1bJukYMxtlZr0kXSzpV6WNBaDcdXrk5O6tZvZtSY9LqpQ0293XlDwZgLLGdesApIrr1gHoVignACFRTgBCopwAhEQ5AQiJcgIQEuUEICTKCUBIlBOAkCgnACFRTgBCopwAhEQ5AQgpsZUwu4PQSyWUYHUIYH9mOf/4PyyOnACERDkBCIlyAhAS5QQgJMoJQEiUE4CQKCcAIVFOAEKinACERDkBCIlyAhAS5QQgJMoJQEiUE4CQKCcAIVFOAELqtJzMbLaZNZvZ6oMRqNzskrQk7RA5tLa26sknn9SePXvSjvIxS5cu1bZt29KOgRLL58jpPyWdV+IcZetBSWdIuuee2WlHOcCmTZs0adIkXXLZ5eEK6sIvflmnnHYmBdXTuXunm6SRklbns292f4+4eULbswnnsopK71M7yBctWuTF2LZtm/fv3z/RbLUjx/h1188sKlcmk/GpX/xiorn6H3eqj51wUlG53N3nzZvnW7duLXqcpD3//PP+zDPPJDpm2ve/du+X7fRIYmuIm9kMSTOSGi+yoyV9RdK7CYz1tKQPKqt1SHWV6uvrixqrrq5OV155pdatW1d0rubmZjU0NKh11+uacOLYosYyM0256CJ98P77ReeSpN/85jfas/1lTTz9/KLHmj59um688UbdfPPNCSRLzuTJk/Xmm2+qpaUl7Sjpaa+19t/EkVPJtvmSD6ys8tWrVyf6W7JYmzdv9po+fX3evHvTjvIxx54wxi+7/Arfs2dP0WNJ8pkzizsyLIWampq9RxWJSfv+1+79stRHTuiar0mauqdVh3zqU2lHOUB9fb3e3LFdhxxySNpRPmbVigZVVVWpooInm3syyimAeHf/NhGLSZJ69eqVdgQcBPm8lOABSf8rabSZbTGzK0ofC0C56/TIyd2nHYwgALA/TtoBhEQ5AQiJcgIQEuUEICTKCUBIlBOAkCgnACFRTgBCopwAhEQ5AQiJcupBtm/frmXLlqUdA0gE5dRDvPHGG/rMZ0/XaWeeHW7JX6BL2lvoqZhNARawyrX1VJlMxo894c+8sqa/Vw0Y7DLzRx99LJGx056zUs1lU1OTX3/9DS7Jhw8f7rfffkcC363itbS0+I033bTvdt4wc6a///77iYyd9px1MJe5e6S9/yhmS/vGlls5ubvPnj3bR4wY4ZL85ptv9pdeeimRcdOes1LN5aOPPuYy8+pDR3hln1ofM26CZzKZBL5jxXnttde8f90grxww2CsHHO59+g/wTZs2JTJ22nNGOZVpObm7X3fddYnfzrTnrJRzOWfOXK/uXePHjRnrO3bsSGTMJKxatcrrDh3sAwYe6suXL09s3LTnrNByYiVMlK1LL52usWP/XEcddZTq6urSjrPPmDFj1Pj75dqzZ49GjRqVdpzUUE4oa2PHjk07Qk5HHnlk2hFSx7N1AEKinACERDkBCIlyAhAS5QQgJMoJQEiUE4CQKCcAIVFOAEKinACERDkBCIlyAhBSp+VkZiPMbLGZrTWzNWZ27cEIBnQ3u3fvTjtCj5LPkVOrpL939xMknSzpW2Z2QmljAd3LihUrdMSwes2fPz/tKD1Gp0umuPtrkl7Lvv22ma2TNFzS2hJnC2vr1q3q1auXBg0alHaUA1RUcJZeqA0bNujoo49WVVXXVw9au3atzjp3kt597z1NmzZNt912W9G5amtrNXv2bA0ZMqTosbqt9lahy7VJGinpZUkDcvzfDEnLs1vqq+vl2pIyctQoP/nkkxMbLynNzc2+cOHCRMdMe85KPZeS/Ic//GFRYyxevNj71A5yVVQkehuXLVuW0K1sk/acdTCXxS3TK6mfpAZJX8pj39RvcKl/oGtraxMbL7K05+xgzOXMmTOLHmfOnLluZv7Tn/40gVSlkfacFVpOeR3Lmlm1pJ9Lus/df5HP5wDl5NJLp+v444/T8ccfn3aUHqPTcjIzk3SPpHXufkvpIwHd06c//em0I/Qo+TyCeqqk6ZLOMbOV2e38EucCUObyebZuiSQ7CFkAYB+eewYQEuUEICTKCUBIlBOAkCgnACFRTgBCopwAhEQ5AQiJcgIQEuUEICTKCUBIlBOAkCinAri7nn76aUnSrl27tGTJknQDocveeuutfXPZ2NioxsbGdAPh49pbha6YTQFW18u1FWv9+vUHjGcVFf7qq68WPa573FUKo27Fmjt33gHjDas/0jOZDPOYzlzm7BGOnAowevRo3TZrlvodeoT6DhysuXPmaujQoWnHQhd8/et/pUsuu1z9hh2tAQMP1a9/+bDa1lVEFNZW9gkPapb8oAlI6rbOf/BB1fSu0ZQpFyUyniTuGAVKYi4zmYxu/fG/6Zyzz9S4ceMSSMU8doW75/ymUU5B8ENdmKhzyTwWrr1y4rQOQEiUE4CQKCcAIVFOAEKinACERDkBCIlyAhAS5QQgJMoJQEiUE4CQKCcAIVFOAELqtJzMrLeZLTWzRjNbY2Y/OBjBAJS3TlclsLY/s+7r7u+YWbWkJZKudffnOvickH8yHvUv2SX+mr1QUeeSeSxcl1clyC7w90723ersFvMn4yBpbW1VJpNJOwYS0NLSknYEtCOvx5zMrNLMVkpqlvSkuz+fY58ZZrbczJYnnDGcr118sa655pq0YyABhx9+uBYuXJh2DOTS3vq9uTZJdZIWSxrTyX6pr0uca0uKJK+trU1svL1jsqUzlzNnzkx0PLaC57L4NcTdfafayum8Qj4PAAqVz7N1g82sLvt2jaTPS1pf4lwAylxVHvsMlTTHzCrVVmYL3P2R0sYCUO46LSd3f0FSMpemAIA88QpxACFRTgBCopwAhEQ5AQiJcgIQEuUEICTKCUBIlBOAkCgnACFRTgBCopwAhEQ5AQiJcgIQUqcXOOjSoD30AgdNTU0aPXr0vvXDq6t7afPmlzVkyJAk4uEguv/+B3TJ9Evk2bkcMfIT2rSxqUdfoCDqbfN2LnCQz3pOyBo5cqSmfOkrerrx/+Qt72vqOado8ODBacdCF5x33hd07PFj1NxrqD7ctFI3fO+6sHfecsWRU4FaW1t1xVXfVL++ffTvP75FFRWcGXdXO3bs0F9Ou0Rf/fJUXTVjRtpxSi5q+bZ35EQ5AWWiu5UTv/YBhEQ5AQiJcgIQEuUEICTKCUBIlBOAkCgnACFRTgBCopwAhEQ5AQiJcgIQEuUEIKS8y8nMKs1shZk9UspAACAVduR0raR1pQpSrl588UX96JZbw62Y8O677+oH/3Szdu3alXaUj5l1x3+osbEx7Rgf87tFizV//oNpx+g53L3TTVK9pKcknSPpkTz294hbRHfeeadL8iuvusYzmUzacfZpampyST7mxAm+c+fOtOMcYPDQeu9XO9BXrlyZdpQDTJt+mcvM7777nrSj5JT2/a+D+2XuHmnvP/zAsnlI0gRJZ1FO7o899liiuayy2nsPGOQLFy4sKtfrr7+e+PdswLGf8cuvuLKoXJlMxocNG5Zorv7jL/DhR40qKpe7+0033ZRort6jxntFRaX/8Y9/LCrXnDlz/Pbb7yj69u0v7ftfB/fLnD3S6WJzZjZZ0vnu/k0zO0vSd9x9co79Zkjau5zghA4HTUlntzVfLS0teuqppxIZb8GCBZp73wM6dvRxevaZxRo4cGBR4z377LOJnIq9/PLLuvrqqzVg4KFa/LsnNH78+KLG27BhgzZu3Fh0LkmafOGFqj6kj27913/RNVdfXdRY27dv19KlSxPJdf3MG7V63QZNuXCyFjxwnyorK7s8Vt++fbV79+5ET/e722Jz+Rw1/bOkLZJekrRV0m5J93byOam3ca4tokWLFvnpZ3/Od+zYkXaUAzQ3N/uEk072hoaGtKN8zJQvf9VvvyPZo4ok/OjWn/i0Sy7z1tbWoseqqalJ/Gc27ftfB/fLrh057a+jI6eP7Jdc3SeokNsKpKlPnz567733yvrIidc5AQiJCxwAAXHkxJETgKAoJwAhUU4AQqKcAIREOQEIiXICEBLlBCAkyglASJQTgJAoJwAhUU4AQqKcepDGxkbdNmsWf0PYQ7m7bps1Sy+88ELaUQ6KqrQDIBkrVqzQ2Z+bpD3VffXCqjW6845ZYf/QE4Vzd33jqqu14NdPqPLD7+u/F/1OY8eOTTtWabW30FMxmwIsYJVr66kymYwPHT7igNt63333JzJ22nPW/eayuLvP22/LTz+7977b+dkza3zHDvm8eQfe/uEjqj2TKWzstOesg7nMGZjTuh7AzPSr//qFzj33XEnS448/rgsuOD/lVOiK1lapudnVe+gnVFN/rF57NaM9e6QLL5Qef7xtny98Qfrlwy3q6QfGlFMPMXHiRE2Y0LZ0+6RJk1RbW5tyInRFXZ307DMf6KhBr+iImk16bskHOuwwqbZWmjSpbZ8JE9q2no7HnIBgBg2SVq/8QJJUVcb30DK+6UBc5VxKe3FaByAkyglASJQTgJAoJwAhUU4AQqKcAIREOQEIiXICEBLlBCAkyglASJQTgJAoJwAh5VVOZvaSma0ys5VmtrzUocrJ7t27tXLlyrRjIAHNzc1qako7Rc9RyJHT2e5+ortPLFmaMnTvvfdqwoSJWrBgQdpRUKS//c4/aNzEajU2pp2kZ2Bhhi5oaGjQxInJdbRVVunyK6/WsGHDdNppp3V5HBaYK9zcufN02WWXJjZeryM+qTPO3qQX17fo8MMTG/YAAwaUZtxo8i0nl/SEmbmkO939ro/uYGYzJM1IMlxURxxxhM4880zt3Lmz6LEaGxtlldWqqqzQoEGDihrru9/9rqZOnVp0pnLyyU8erXHjximTyRQ9VmNjo/yDd3ToYRXq3TuBcDmsWSMdc0xpxg4nn5XRJQ3P/nu4pEZJZ3Syf+qLpufaIpo7d673G1DnDQ0NaUfJKe05605z+Y2rrvFRxxzir79ekuuGFL2lPWcdzGXOwOYFXuPMzL4v6R13/1EH+xQ26EFS6G09WN5++231798/7Rg5Rb28VMS5zGQy2r27Uv36pZ0kt6BTKXfPmazTB8TNrK+Z9d/7tqRJklYnG6+8RS0mFKaioiJsMXVH+TzmNETSw9nfoFWS7nf335Y0FYCyV/BpXV6DclrXY3BaV6iY3y+pB57WAUAaKCcAIVFOAEKinACERDkBCIlyAhAS5QQgJMoJQEiUE4CQKCcAIVFOAEKinACERDkBCKms1hCP+hf2KBxz2fNx5AQgJMoJQEiUE4CQKCcAIVFOAEKinACERDkBCIlyAhAS5QQgJMoJQEiUE4CQKCcAIVFOAEKinACERDkBCCmvcjKzOjN7yMzWm9k6Mzul1MEAlLd8F5v7iaTfuvtXzKyXpD4lzAQAMnfveAezWkkrJX3CO9v5T5+T134A4O45lzXN57RulKQ3JP3MzFaY2d1m1vejO5nZDDNbbmbLi8wKAHkdOU2U9JykU939eTP7iaS33P2mDj6HIycAeSnmyGmLpC3u/nz2/YckjU8qGADk0mk5uftWSZvNbHT2Q+dKWlvSVADKXqendZJkZidKultSL0kbJV3u7m92sD+ndQDy0t5pXV7lVCjKCUC+innMCQAOOsoJQEiUE4CQKCcAIVFOAEKinACERDkBCIlyAhAS5QQgJMoJQEiUE4CQKCcAIVFOAELK9wIHhdomaVNCYx2WHS8achUmai4pbrZyyHVUe/9RkiVTkmRmy919Yto5PopchYmaS4qbrdxzcVoHICTKCUBI3aGc7ko7QDvIVZiouaS42co6V/jHnACUp+5w5ASgDFFOAEKinACERDkBCIlyAhDS/wMCuQvLRAlkPQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 345.6x345.6 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "map = (env.map == -np.ones_like(env.map)).astype(int)\n",
    "map[1, 1] = 2\n",
    "map[-2, -2] = 3\n",
    "plt.pyplot.matshow(map, cmap=plt.colors.ListedColormap([\"white\", \"black\", \"red\", \"yellow\"]))\n",
    "N = 0\n",
    "for i in range(map.shape[0] - 2):\n",
    "    for j in range(map.shape[0] - 2):\n",
    "        x = j\n",
    "        y = i\n",
    "        if not map[i + 1, j + 1] == 1:\n",
    "            ox = 0\n",
    "            oy = 0\n",
    "            dir = [\n",
    "                (0 + ox, -1 + oy),\n",
    "                (1 + ox, 0 + oy),\n",
    "                (0 + ox, 1 + oy),\n",
    "                (-1 + ox, 0 + oy),\n",
    "            ][policy[N]]\n",
    "\n",
    "            plt.pyplot.arrow(\n",
    "                1 + x + (0.15 if i == j == map.shape[0] - 3 else 0),\n",
    "                1 + y,\n",
    "                dir[0] / 2,\n",
    "                dir[1] / 2,\n",
    "                head_width=0.1,\n",
    "                head_length=0.05\n",
    "            )\n",
    "            N += 1\n",
    "# plt.colorbar()\n",
    "plt.pyplot.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random Teleport"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class four_room_random():\n",
    "\n",
    "\tdef __init__(self):\n",
    "\t\tself.nS = 20\n",
    "\t\tnS = self.nS\n",
    "\t\tself.nA = 4\n",
    "\n",
    "\t\tself.map = [[-1, -1, -1, -1, -1, -1, -1],\n",
    "\t\t\t\t\t[-1,  0,  1,  2,  3,  4, -1],\n",
    "\t\t\t\t\t[-1,  5,  6, -1,  7,  8, -1],\n",
    "\t\t\t\t\t[-1,  9, -1, -1, 10, -1, -1],\n",
    "\t\t\t\t\t[-1, 11, 12, 13, 14, 15, -1],\n",
    "\t\t\t\t\t[-1, 16, 17, -1, 18, 19, -1],\n",
    "\t\t\t\t\t[-1, -1, -1, -1, -1, -1, -1]]\n",
    "\t\tmap = np.array(self.map)\n",
    "\n",
    "\t\t# We build the transitions matrix P using the map.\n",
    "\t\tself.P = np.zeros((nS, 4, nS))\n",
    "\t\tfor s in range(nS):\n",
    "\t\t\ttemp = np.where(s == map)\n",
    "\t\t\tx, y = temp[0][0], temp[1][0]\n",
    "\t\t\tup = map[x-1, y]\n",
    "\t\t\tright = map[x, y+1]\n",
    "\t\t\tdown = map[x+1, y]\n",
    "\t\t\tleft = map[x, y-1]\n",
    "\n",
    "\t\t\t# Action 0: go up.\n",
    "\t\t\ta = 0\n",
    "\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\t# Up\n",
    "\t\t\tif up == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.7\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, up] += 0.7\n",
    "\t\t\t# Right\n",
    "\t\t\tif right == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, right] += 0.1\n",
    "\t\t\t# Left\n",
    "\t\t\tif left == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, left] += 0.1\n",
    "\t\t\t\n",
    "\t\t\t# Action 1: go right.\n",
    "\t\t\ta = 1\n",
    "\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\t# Up\n",
    "\t\t\tif up == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, up] += 0.1\n",
    "\t\t\t# Right\n",
    "\t\t\tif right == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.7\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, right] += 0.7\n",
    "\t\t\t# Down\n",
    "\t\t\tif down == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, down] += 0.1\n",
    "\t\t\t\n",
    "\t\t\t# Action 2: go down.\n",
    "\t\t\ta = 2\n",
    "\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\t# Right\n",
    "\t\t\tif right == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, right] += 0.1\n",
    "\t\t\t# Down\n",
    "\t\t\tif down == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.7\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, down] += 0.7\n",
    "\t\t\t# Left\n",
    "\t\t\tif left == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, left] += 0.1\n",
    "\n",
    "\t\t\t# Action 3: go left.\n",
    "\t\t\ta = 3\n",
    "\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\t# Up\n",
    "\t\t\tif up == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, up] += 0.1\n",
    "\t\t\t# Down\n",
    "\t\t\tif down == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.1\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, down] += 0.1\n",
    "\t\t\t# Left\n",
    "\t\t\tif left == -1:\n",
    "\t\t\t\tself.P[s, a, s] += 0.7\n",
    "\t\t\telse:\n",
    "\t\t\t\tself.P[s, a, left] += 0.7\n",
    "\t\t\t\n",
    "\t\t\t# Set to teleport back when in the rewarding state\n",
    "\t\t\tprob = 1/(self.nS - 1)\n",
    "\t\t\tif s == self.nS - 1:\n",
    "\t\t\t\tfor a in range(4):\n",
    "\t\t\t\t\tfor ss in range(self.nS):\n",
    "\t\t\t\t\t\tself.P[s, a, ss] = 0\n",
    "\t\t\t\t\t\tif ss != s:\n",
    "\t\t\t\t\t\t\tself.P[s, a, ss] = prob\n",
    "\n",
    "\t\t\t\n",
    "\t\t# We build the reward matrix R.\n",
    "\t\tself.R = np.zeros((nS, 4))\n",
    "\t\tfor a in range(4):\n",
    "\t\t\tself.R[nS - 1, a] = 1\n",
    "\n",
    "\t\t# We (arbitrarily) set the initial state in the top-left corner.\n",
    "\t\tself.s = 0\n",
    "\n",
    "\t# To reset the environment in initial settings.\n",
    "\tdef reset(self):\n",
    "\t\tself.s = 0\n",
    "\t\treturn self.s\n",
    "\n",
    "\t# Perform a step in the environment for a given action. Return a couple state, reward (s_t, r_t).\n",
    "\tdef step(self, action):\n",
    "\t\tnew_s = np.random.choice(np.arange(self.nS), p=self.P[self.s, action])\n",
    "\t\treward = self.R[self.s, action]\n",
    "\t\tself.s = new_s\n",
    "\t\treturn new_s, reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_ran = four_room_random()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([5.00886987, 5.15621899, 5.34566785, 5.51406722, 5.38513633,\n",
       "        5.15621899, 5.00886987, 5.7249428 , 5.53511708, 5.34566785,\n",
       "        5.92046093, 5.51406722, 5.7249428 , 5.92046093, 6.08886098,\n",
       "        6.25726077, 5.38513633, 5.53511708, 6.25726077, 6.44971743]),\n",
       " array([1, 1, 1, 2, 2, 2, 0, 2, 3, 2, 2, 1, 1, 1, 1, 2, 1, 0, 1, 0]),\n",
       " 0.11787970699374473,\n",
       " array([0.        , 0.14734912, 0.33679799, 0.50519735, 0.37626646,\n",
       "        0.14734912, 0.        , 0.71607294, 0.52624721, 0.33679799,\n",
       "        0.91159106, 0.50519735, 0.71607294, 0.91159106, 1.07999111,\n",
       "        1.2483909 , 0.37626646, 0.52624721, 1.2483909 , 1.44084756]))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "V, policy, gain, bias= VI_ar(env_ran)\n",
    "V, policy, gain, bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAECCAYAAADXWsr9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAR/UlEQVR4nO3de3BW9Z3H8ff3SUiQCESqVSeA6E5xt7auYlCpiIq6g1W31d1ZtV66WsedqWVoLeOq3fXWrXVxceiu1opgUW7Vik4dxsuyXtbFWRohynIJBi8JCaKxK3cBSfLdP3KkWYjkec5zOSf5fV4zZ0jCc873+5zkk/OcJ+d3fubuiEj/lkm6AREpPgVdJAAKukgAFHSRACjoIgFQ0EUCkKqgm9kkM3vbzN4xs1tKXPtRM2szs9WlrNut/ggze8XM1prZGjObUuL6A82szsxWRvXvKmX9qIcyM3vTzBaXunZUv8nMVpnZW2a2vMS1q83sKTNbZ2YNZjauoNtPy9/RzawMaATOB1qBN4Ar3H1tiepPAHYAj7v710pRc7/6RwNHu3u9mQ0GVgDfLuHzN6DK3XeY2QBgKTDF3ZeVon7Uw01ALTDE3S8qVd1u9ZuAWnf/QwK1HwP+y91nmVkFMMjdtxRq+2k6op8KvOPu77n7Z8BvgG+Vqri7vwZ8Uqp6PdTf5O710cfbgQagpoT13d13RJ8OiJaSHQXMbDhwITCrVDXTwsyGAhOA2QDu/lkhQw7pCnoN0NLt81ZK+IOeJmY2CjgZ+H2J65aZ2VtAG7DE3UtZfwZwM9BZwpr7c+DfzWyFmd1QwrrHAh8Dv45OXWaZWVUhC6Qp6AKY2aHAIuCH7r6tlLXdvcPdTwKGA6eaWUlOYczsIqDN3VeUot5BjHf3McAFwI3R6VwplANjgIfc/WRgJ1DQ96jSFPSNwIhunw+PvhaM6Nx4ETDf3Z9Oqo/oZeMrwKQSlTwD+MvoHPk3wEQzm1ei2vu4+8bo3zbgGbpOJ0uhFWjt9grqKbqCXzBpCvobwFfM7NjozYjLgWcT7qlkojfDZgMN7n5/AvWPMLPq6OND6HpTdF0parv7re4+3N1H0fV9f9ndrypF7c+ZWVX0JijRy+a/AEryFxh3/xBoMbPjoy+dCxT0TdjyQm4sH+7ebmY/AF4EyoBH3X1Nqeqb2ULgbOBwM2sF7nD32aWqT9dR7WpgVXSeDHCbuz9XovpHA49Ff/3IAE+6eyJ/5krIkcAzXb9vKQcWuPsLJaw/GZgfHeTeA64t5MZT8+c1ESmeNL10F5EiUdBFAqCgiwRAQRcJgIIuEoBUBr3Elx+mprbqq36x6qcy6ECSOzvRb7Tqq34xNprWoItIARXlghkz01U4Iglxd9v/azqiiwRAQRcJgIIuEgAFXSQACrpIABR0kQAo6CIBUNBFAqCgiwRAQRcJgIIuEoCsgp7k5Icikr9eB7XEmfxQg1pEkhN3UEuikx+KSP6yCbomPxTp4wo2U0t0C5yk784hIj3IJuhZTX7o7jOBmaBzdJG0yeale9CTH4r0B70e0ZOe/FBE8qd7xon0M7pnnEigFHSRACjoIgFQ0EUCoKCLBEBBFwmAgi4SAAVdJAAKukgAFHSRABRsmGqaFOOyXpFsmR1wBWridEQXCYCCLhIABV0kAAq6SAAUdJEAKOgiAVDQRQKgoIsEQEEXCYCCLhIABV0kAAq6SAB6DbqZPWpmbWa2uhQNiUjhZXNEnwNMKnIf/UbSI+dCry896zXo7v4a8EkJeunzWltbqRl5DC+99FIi9evr6xkx6jhWrlyZSP3nn3+ePxn9pzQ1NSVSXw7C3XtdgFHA6mweGz3ek1ziam9v94svvjjv+oOGVPvq1atzrr9z504/pbY27/pDDjvcN23alHP9trY2HzFiRN71j6oZ4bt27cq5/vr1672uri7n9QrljTfe8MbGxry3k4Kf/wMyqfnRu8lkMpxwwgmsXbs21vptbW1s376dyooKBg0alPP6FRUV/PmJJ7Jl8+ZY9d99910ABg8ZTEVFRc7rV1VVMWbMmFjrdq//pcOPIJPJ/X3eSy+9lFWrViX28n/s2LGccMIJrF7dD9+O6in9+y8EckTP10cffeTjzzon1tG8EBoaGnzCxPO8paUlkfpLly718yZd6Js3b461/siRIxP9/gFeU1NTkO0k/PN/QCazmk3VzEYBi939a70+mORnU83mOUn6HHPMMWzYsCGx75+ZUVNTQ2tra97bSZLHmU3VzBYC/w0cb2atZva9YjQnIsXT6zm6u19RikZEpHh0ZZxIABR0kQAo6CIBUNBFAqCgiwRAQRcJgIIuEgAFXSQACnqKdHR0sG3btqTbkH5IQU+Jjo4OrrjqGo748pGJjWeXfqynkS75LiQ/eqfPueOuu/f1X1E50D/44IPY2+qL+/+++2fsW//vb/vH2M89rlt+cvu++vfdPyOvbaVg/x+QSR3RU+LKKy7nggsuAODR2bM46qijEu6otNatWwdmYBmWL19e8hFsK1asAMuAWez7EaRZVsNUc96ohqnG8sADDzB58uS8+0/BMMmc1+ns7OTvvv8D2to+5okFcxk4cGAROvtie/bs4fKrruFLw4Yx86EHY90443Mp2P8HNFCwO8yI5COTyfDIr36ZWP3Kykqe+e0TidUvNr10FwmAgi4SAAVdJAAKukgAFHSRACjoIgFQ0EUCoKCLBEBBFwlANhM4jDCzV8xsrZmtMbMppWhMRAonmyN6O/Bjd/8qcDpwo5l9tbhtieRu9+7dSbeQWtnMj77J3eujj7cDDUBNsRsTyUVzczPHfuV4pk+fnnQrqZTTOXo02eLJwO+L0k3Cdu7cybRp0xKrP2zYsMRqp0FdXR2LFy+Ote5ZE8/nw9YNTJ06FTOLvaxatarAzyodsh6mamaHAv8J/Mzdn+7h/7vPj35KwTqMIe4wz/r6ek455ZTEhrm6Oxs3bmT48OF5bScFwyRjrTd69GjWr18fa/1rrv0ec+c8SmVlJZdddlms+kOHDuXee++NNbd9dynY//GGqZrZAGARML+nkEcbnwnMjB7fNweEJ8zM8g55X7Znz57Y686Z/QjDhg3j4m9O4txzzy1gV/1Dr0G3rl9Ps4EGd7+/+C2J5C6TyTBj+n1Jt5Fa2ZyjnwFcDUw0s7ei5ZtF7ktECiib+dGXAsmedIhIXnRlnEgAFHSRACjoIgFQ0EUCoKCLBEBBFwmAgi4SAAVdJAAKukgAFHSRACjokSVL/oO/+pvLATj1jAm0tLTE3lY+46ELsfRFN918Kxs2bADgO9+9Lq9taf8fSLOpRqqrh/Lhh5soq6rmg42tlJWVJd1SUA4dNJCyARWQKSeD4+6pDU2f5O4FXwBPcolr2bJlfsFF3/KWlpbY2/CuHRD0EtfP7rnXJ//ox97R0aH9n9/+PyCTWd9hJhdJ33iiGM8pF6EfibT/k+U93GFG5+giAVDQRQKgoIsEQEEXCYCCLhIABV0kAAq6SAAUdJEAKOgiAchmfvSBZlZnZiuj+dHvKkVjIlI4vV4CG03JVOXuO6I52JYCU9x92UHW0SWwAdP+T1ZPl8BmM1OLAzuiTwdES7LfSRHJSVbn6GZWZmZvAW3AEnfvl/Ojb9myheuvvz7pNoL13HPPMWvW7KTb6JdyGr1mZtXAM8Bkd1+93/9pfvSIXjrG23+jRo2iublZ+z9PeY9ec/ctwCvApB7+b6a717p7bewOJWhJn9v3Z9m8635EdCTHzA4BzgfWFbkvESmgbG4ldTTwmJmV0fWL4Ul3X1zctkSkkLJ51/1/gJNL0IuIFImujBMJgIIuEgAFXSQACrpIABR0kQAo6CIBUNBFAqCgiwRAQRcJgIIuEgAFPfLUokWcdc65AAw/5jjefvvt2NvqaTbLkJY4rrj6b/fNjz5h4vl5jWTri8+/2BT0yIlf/zoVlZVUHHYU1YcdxpFHHpl0S0E5+8xvMGBgFRWHVjP+G6cn3U6/o2mTu2lsbGTGvz7APf90N9XV1YVtSno1b/582j7+X340ZXKfvnlE0r33dOMJBV2kwNIYdL10FwmAgi4SAAVdJAAKukgAFHSRACjoIgFQ0EUCoKCLBEBBFwlA1kGPJlp808w0eYNIH5PLEX0K0FCsRvqDzs5Oli1bltgluHv37qWuri6R2gC7du2ivr4+sfpbt25l9erVvT8wQNlOmzwcuBCYVdx2+rb169czbtw4vj95SiJhf/XVVznttNO4d9q0ktcGWLhwIbVjxzJv3vxE6s+YMYMxtWN56aWXEqmfalmOr32KrqmQzwYWZ/F4T3KJa+/evQWpP2BglS9ZsiTn+lu3bi1IfcuUeUNDQ871m5ubC1Q/41u3bs25fl1dXUHqDx5anXNtd/e77r7bZ8+eHWvd7lLw839AJnude83MLgLa3H2FmZ19kMd1nx+9TyovL6e1tZXNmzfHWv+dd97hkksuYfz48Zx55pk5rz9kyBDef/99duzYEav+a6+9xo033sh3rryK0aNH57z+yJEjaWxsZM+ePbHqL1y4kHvuuYdbbr2NwYMH57x+bW0ta9asobOzM1b96dOnM2fOHP753p/HWv+O22+npqaG6667Ltb6qdZT+v3/H51/DrQCTcCHwKfAvF7WSfo3WiI+/fRT/9XDD/vu3bsTqf/JJ5/4I7NmeUdHRyL1N27c6I/PneudnZ2J1F+/fr0vevrp2OsDXlNTk3cfKfj5PyCTOY1Hj47oU939ol4el/1GiyCX5yTyOTOjpqaG1tbWvLeTJNd4dJEw9XqO3p27vwq8WpRORKRodEQXCYCCLhIABV0kAAq6SAAUdJEAKOgiAVDQRQKgoIsEQEFPkQ0bNjBv3jxdwpsS7s6CBQtoampKupW85XRlnBRPc3Mzp4+fwLZPP+P1ZXX88t9+kfg10yFzd35401RmPb6AwYcMYNnS1xg1alTSbcXX00iXfBeSH73T59z6k3/Y138mk/FNmzbF3pb2f+4/tqP/rGJf/2dNrPS2NjxT9sfnNPXm8qy3lYL9f2AmY+0VBb3g2tvbffr06Q7kPcxV+z/3H9uHHzYfVF3lhwyu9N/9rutru3d3PZ+f/hRvb89+WynY/7nfeEJKo6ysjIqKCgAqKysT7iY8N9zgHHHEpwwd6kyc2PW1z78NFRVQVpZcb4WgoItELrnEk26haPSuu0gAFHSRACjoIgFQ0EUCoKCLBEBBFwmAgi4SAAVdJAAKukgAsroyzsyagO1AB9Du7rXFbEpECiuXI/o57n6SQv7F9uzZw5NPPkl7e3vSrQSppaVFUyZ/AV3rvp9du3axd+/eWOs2NjZy2WWXcfG3L+Xp3z5Bebl2b662bdsWe90HH3yQadPuY+5cuPLKAjbVH2Qz7g54H6gHVgA3fMFjbgCWR0vSw/RiKeT86C+++GLO9evq6vz0ceNi9/+5vrr/CzU/etXg7MeO97acOQF//fXc1knB/o83Hh2oif79MrASmNDL45N+oolYt26dA37l1d9NbOpi974b9Hzdeeednikv92efLcptFrJeUrD/D2gqp2mTAczsTmCHu//LQR6T20YLLNfnVCjt7e28/PLLnHfeeWQyyf1BI+lbUCW1/9va2mhqauLUU09LpP7nkr4DmPcwbXKvQTezKiDj7tujj5cAd7v7CwdZJ8igp0WoQf+jZJ9/GoOezbtFRwLPRD885cCCg4VcRNIn55fuWW1UR/RE6YiuI/r+X9OVcSIBUNBFAqCgiwRAQRcJgIIuEgAFXSQACrpIABR0kQAo6CIBUNBFAtAv74yQ9CWgodP+Tx8d0UUCoKCLBEBBFwmAgi4SAAVdJAAKukgAFHSRACjoIgFQ0EUCoKCLBEBBFwlAVkE3s2oze8rM1plZg5mNK3ZjIlI42Q5q+QXwgrv/tZlVAIOK2JOIFFg2UzINBd4CjvMs78yf9AQOIiGLO4HDscDHwK/N7E0zmxXNwSYifUQ2QS8HxgAPufvJwE7glv0fZGY3mNlyM1te4B5FJE/ZvHQ/Cljm7qOiz88EbnH3Cw+yjl66iyQk1kt3d/8QaDGz46MvnQusLXBvIlJEWc2mamYnAbOACuA94Fp333yQx+uILpKQno7o/XLaZJGQadpkkUAp6CIBUNBFAqCgiwRAQRcJgIIuEgAFXSQACrpIABR0kQAo6CIBUNBFAlCs+dH/ADTnsf7h0TaSkGRt1Vf9fOsf09MXizKoJV9mttzda0OrrfqqX6z6eukuEgAFXSQAaQ36zEBrq77qF6V+Ks/RRaSw0npEF5ECUtBFAqCgiwRAQRcJgIIuEoD/A/hGITfJaIrVAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "map = (env.map == -np.ones_like(env.map)).astype(int)\n",
    "map[-2, -2] = 3\n",
    "plt.pyplot.matshow(map, cmap=plt.colors.ListedColormap([\"white\", \"black\", \"yellow\"]))\n",
    "N = 0\n",
    "for i in range(map.shape[0] - 2):\n",
    "    for j in range(map.shape[0] - 2):\n",
    "        x = j\n",
    "        y = i\n",
    "        if not map[i + 1, j + 1] == 1:\n",
    "            ox = 0\n",
    "            oy = 0\n",
    "            dir = [\n",
    "                (0 + ox, -1 + oy),\n",
    "                (1 + ox, 0 + oy),\n",
    "                (0 + ox, 1 + oy),\n",
    "                (-1 + ox, 0 + oy),\n",
    "            ][policy[N]]\n",
    "\n",
    "            plt.pyplot.arrow(\n",
    "                1 + x + (0.15 if i == j == map.shape[0] - 3 else 0),\n",
    "                1 + y,\n",
    "                dir[0] / 2,\n",
    "                dir[1] / 2,\n",
    "                head_width=0.1,\n",
    "                head_length=0.05\n",
    "            )\n",
    "            N += 1\n",
    "# plt.colorbar()\n",
    "plt.pyplot.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}