task 4 HA 3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = [\"s\", \"a\", \"r\", \"s+1\"]\n",
    "df1 = pd.read_csv(r\"\\data_policy1.csv\", names=headers)\n",
    "df2 = pd.read_csv(r\"\\data_policy2.csv\", names=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def td1 (p,T):\n",
    "    \n",
    "    gamma = 0.95\n",
    "    v = [0,0,0,0,0]\n",
    "    v_values = [[0 for x in range(5)] for y in range(len(p))]\n",
    "    \n",
    "    for t in range(T):\n",
    "        v_s1 = [0,0,0,0,0]\n",
    "        alpha = (10/(t**(2/3)+1))\n",
    "        \n",
    "        for i in range(len(v)):\n",
    "            if (i == p['s'][t]):\n",
    "                v_s1[i] = v[i]+alpha*(p['r'][t] + gamma*v[p['s+1'][t]] - v[i])\n",
    "            else:\n",
    "                v_s1[i] = v[i]\n",
    "        \n",
    "        v = v_s1\n",
    "        v_values[t][:] = v\n",
    "    \n",
    "    return v_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "def td2(p,T):\n",
    "    gamma = 0.95\n",
    "    v = [0,0,0,0,0]\n",
    "    n_t = [[0 for x in range(2)] for y in range(len(v))]\n",
    "    v_values = [[0 for x in range(5)] for y in range(len(p))]\n",
    "\n",
    "    for t in range(T):\n",
    "        v_s1 = [0,0,0,0,0]\n",
    "        for i in range(len(v)):\n",
    "            n_t[p['s'][t]][p['a'][t]] += 1\n",
    "            alpha = (10/(n_t[i][p['a'][t]]**(2/3)+1))\n",
    "            if (i == p['s'][t]):\n",
    "                v_s1[i] = v[i] + alpha * (p['r'][t] + gamma*v[p['s+1'][t]]-v[i])\n",
    "            else:\n",
    "                v_s1[i] = v[i]\n",
    "        \n",
    "        v = v_s1\n",
    "        v_values[t][:] = v\n",
    "    \n",
    "    return v_values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mb(p,T):\n",
    "    gamma = 0.95\n",
    "    states = 5\n",
    "    v_values = [[0 for x in range(5)] for y in range(len(p))]\n",
    "    n_s = np.zeros(states)\n",
    "    n_s_s = np.zeros((states,states))\n",
    "    sum_r = np.zeros(states)\n",
    "    v = np.zeros(states)\n",
    "    alpha = 1/states\n",
    "\n",
    "    for t in range(T):\n",
    "        r_hat = np.zeros(states)\n",
    "        p_hat = np.zeros((states,states))\n",
    "        sum_r[p['s'][t]] += p['r'][t]\n",
    "        n_s[p['s'][t]] += 1\n",
    "        n_s_s[p['s'][t]][p['s+1'][t]] +=1\n",
    "\n",
    "        for i in range(states):\n",
    "            r_hat[i] = (alpha + sum_r[i])/(n_s[i] + alpha)\n",
    "            for a in range(states):\n",
    "                p_hat[i][a] = (n_s_s[i][a] + alpha)/(n_s[i] + alpha* states)\n",
    "\n",
    "        v = np.linalg.inv(np.identity(states) - gamma * p_hat) @ r_hat\n",
    "        v_values[t][:] = v\n",
    "    return v_values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "td1_p1 = td1(df1,len(df1))\n",
    "td1_p2 = td1(df2,len(df2))\n",
    "td2_p1 = td2(df1,len(df1))\n",
    "td2_p2 = td2(df2,len(df2))\n",
    "mb_p1 = mb(df1,len(df1))\n",
    "mb_p2 = mb(df2,len(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.8726134669318023, 5.232601315444726, 7.253938064075787, 8.492713247209588, 9.81056159281165]\n",
      "[0.4090530945007664, 0.38215038034959636, 0.4192075633622319, 0.5812207409361677, 1.3368578627055732]\n",
      "[5.974780392260192, 6.617697832968905, 7.520237459704777, 8.630636601550751, 9.946323695347846]\n",
      "[0.3892795343915126, 0.3502723507653677, 0.35884334419681824, 0.43130581750643826, 1.2806172074595281]\n",
      "[6.115860858300185, 6.708746468621741, 7.594892598170689, 8.760584035601639, 10.063296898829226]\n",
      "[0.41487126860979295, 0.3917415427923737, 0.43289612597412025, 0.6550519674935147, 1.388359173459151]\n"
     ]
    }
   ],
   "source": [
    "print(td1_p1[len(dp1)-1][:])\n",
    "print(td1_p2[len(dp1)-1][:])\n",
    "print(td2_p1[len(dp1)-1][:])\n",
    "print(td2_p2[len(dp1)-1][:])\n",
    "print(mb_p1[len(dp1)-1][:])\n",
    "print(mb_p2[len(dp1)-1][:])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "c7a68dc2e3110d80647fac8693cdc5243ade541bc79b3bfddbf1397505f434ab"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}