code_submission_KP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gymnasium in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (1.0.0)\n",
      "Requirement already satisfied: stable-baselines3 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (2.4.1)\n",
      "Requirement already satisfied: torch in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (2.5.1+cu118)\n",
      "Requirement already satisfied: numpy>=1.21.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gymnasium) (1.26.4)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gymnasium) (3.1.1)\n",
      "Requirement already satisfied: typing-extensions>=4.3.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gymnasium) (4.12.2)\n",
      "Requirement already satisfied: farama-notifications>=0.0.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from gymnasium) (0.0.4)\n",
      "Requirement already satisfied: pandas in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (2.2.3)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (3.10.0)\n",
      "Requirement already satisfied: opencv-python in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (4.11.0.86)\n",
      "Requirement already satisfied: pygame in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (2.6.1)\n",
      "Requirement already satisfied: tensorboard>=2.9.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (2.18.0)\n",
      "Requirement already satisfied: psutil in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (6.1.1)\n",
      "Requirement already satisfied: tqdm in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (4.67.1)\n",
      "Requirement already satisfied: rich in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (13.9.4)\n",
      "Requirement already satisfied: ale-py>=0.9.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (0.10.1)\n",
      "Requirement already satisfied: pillow in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stable-baselines3[extra]) (11.1.0)\n",
      "Requirement already satisfied: filelock in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch) (3.17.0)\n",
      "Requirement already satisfied: networkx in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch) (3.4.2)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch) (3.1.5)\n",
      "Requirement already satisfied: fsspec in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch) (2024.12.0)\n",
      "Requirement already satisfied: sympy==1.13.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch) (1.13.1)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from sympy==1.13.1->torch) (1.3.0)\n",
      "Requirement already satisfied: absl-py>=0.4 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (2.1.0)\n",
      "Requirement already satisfied: grpcio>=1.48.2 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.70.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (3.7)\n",
      "Requirement already satisfied: packaging in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (24.2)\n",
      "Requirement already satisfied: protobuf!=4.24.0,>=3.19.6 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (5.29.3)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (65.5.0)\n",
      "Requirement already satisfied: six>1.9 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.17.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (0.7.2)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (3.1.3)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jinja2->torch) (3.0.2)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (1.3.1)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (4.55.6)\n",
      "Requirement already satisfied: kiwisolver>=1.3.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (1.4.8)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (3.2.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas->stable-baselines3[extra]) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas->stable-baselines3[extra]) (2025.1)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from rich->stable-baselines3[extra]) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from rich->stable-baselines3[extra]) (2.19.1)\n",
      "Requirement already satisfied: colorama in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tqdm->stable-baselines3[extra]) (0.4.6)\n",
      "Requirement already satisfied: mdurl~=0.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from markdown-it-py>=2.2.0->rich->stable-baselines3[extra]) (0.1.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install gymnasium stable-baselines3[extra] stable-baselines3 torch\n",
    "# pip install stable-baselines3[extra]\n",
    "# pip install stable-baselines3\n",
    "# pip install torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import urllib.request\n",
    "\n",
    "class GymAPI:\n",
    "\n",
    "    def __init__(self, url):\n",
    "        self.url = url\n",
    "\n",
    "    def new_game(self):\n",
    "        req = urllib.request.Request(f'{self.url}/new_game', method ='POST')\n",
    "        with urllib.request.urlopen(req, timeout=60) as resp:\n",
    "            json_data = resp.read()\n",
    "            json_dict = json.loads(json_data)\n",
    "            return json_dict\n",
    "    \n",
    "    def reset(self, uuid):\n",
    "        req = urllib.request.Request(f'{self.url}/reset/{uuid}', method = 'POST')\n",
    "        with urllib.request.urlopen(req, timeout=60) as resp:\n",
    "            json_data = resp.read()\n",
    "            json_dict = json.loads(json_data)\n",
    "            return json_dict\n",
    "        \n",
    "    def step(self, uuid, action):\n",
    "        data = json.dumps({'action':action}).encode()\n",
    "        req = urllib.request.Request(f'{self.url}/step/{uuid}', data=data, headers={'Content-Type': 'application/json'}, method = 'POST')\n",
    "        with urllib.request.urlopen(req, timeout=60) as resp:\n",
    "            json_data = resp.read()\n",
    "            json_dict = json.loads(json_data)\n",
    "            return json_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New game started {'action_space': {'class': 'Discrete', 'n': 4}, 'info': 'The uuid needs to be included in each call if you want to make an action in this environment. The values of observation and action space describe the objects of gymnasium.spaces, you are expected to create them yourself on your custom Wrapper', 'observation_space': {'class': 'Box', 'dtype': 'numpy.float32', 'high': 4, 'low': 0, 'shape': '(2,)'}, 'uuid': 'afa8031c-94ba-489a-9e42-626da060ba00'}\n",
      "Game reset, initial observation {'observation': [0.0, 0.0]}\n",
      "Result after step: {'done': False, 'info': {}, 'observation': [0, 0], 'reward': -1, 'truncated': False}\n"
     ]
    }
   ],
   "source": [
    "# api = GymAPI(\"http://#.##.###.###:####\")\n",
    "\n",
    "# try:\n",
    "#     game_info = api.new_game()\n",
    "#     print('New game started', game_info)\n",
    "\n",
    "#     uuid = game_info['uuid']\n",
    "\n",
    "#     init_obs = api.reset(uuid)\n",
    "#     print('Game reset, initial observation', init_obs)\n",
    "\n",
    "#     action=1\n",
    "#     result = api.step(uuid,action)\n",
    "#     print('Result after step:', result)\n",
    "\n",
    "# except Exception as e:\n",
    "#     print('Error during API interaction:', e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gymnasium as gym\n",
    "from gymnasium.spaces import Box, Discrete\n",
    "from typing import Optional\n",
    "\n",
    "class MazeEnv(gym.Env):\n",
    "    \n",
    "    def __init__(self,api):\n",
    "        super(MazeEnv, self).__init__()\n",
    "        self.api = api\n",
    "        self.uuid = None\n",
    "\n",
    "        maze_game = self.api.new_game()\n",
    "        self.uuid = maze_game['uuid']\n",
    "\n",
    "        self.observation_space = gym.spaces.Dict(\n",
    "                {\n",
    "                    'agent': gym.spaces.Box(0.0, 4.0, shape=(2,), dtype=np.float32),\n",
    "               }\n",
    "            ) \n",
    "\n",
    "        self.action_space = gym.spaces.Discrete(4)\n",
    "\n",
    "\n",
    "    def reset(self, seed: Optional[int] = None, options: Optional[dict] = None):\n",
    "        \n",
    "        super().reset(seed=seed)\n",
    "\n",
    "        response = self.api.reset(self.uuid)\n",
    "        observation = {\n",
    "        \"agent\": np.array(response[\"observation\"], dtype=np.float32)\n",
    "                        }\n",
    "\n",
    "        return observation, {}\n",
    "    \n",
    "    def step(self, action:int):\n",
    "\n",
    "        action = int(action)\n",
    "        \n",
    "        response = self.api.step(self.uuid, action)\n",
    "       \n",
    "        observation = {\n",
    "        \"agent\": np.array(response[\"observation\"], dtype=np.float32)\n",
    "                }\n",
    "        reward = response['reward']\n",
    "        done = response['done']\n",
    "        truncated = response['truncated']\n",
    "        info = response['info']        \n",
    "\n",
    "        return observation, reward, done, truncated, info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial observation: {'agent': array([0., 0.], dtype=float32)}\n",
      "Step result: Observation={'agent': array([1., 0.], dtype=float32)}, Reward=-0.04, Done=False, Truncated=False\n",
      "Step result: Observation={'agent': array([1., 0.], dtype=float32)}, Reward=-1, Done=False, Truncated=False\n",
      "Step result: Observation={'agent': array([1., 0.], dtype=float32)}, Reward=-1, Done=False, Truncated=False\n",
      "Step result: Observation={'agent': array([0., 0.], dtype=float32)}, Reward=-0.04, Done=False, Truncated=False\n",
      "Step result: Observation={'agent': array([0., 0.], dtype=float32)}, Reward=-1, Done=False, Truncated=False\n",
      "Step result: Observation={'agent': array([1., 0.], dtype=float32)}, Reward=-0.04, Done=False, Truncated=False\n",
      "Step result: Observation={'agent': array([2., 0.], dtype=float32)}, Reward=-0.04, Done=False, Truncated=False\n",
      "Step result: Observation={'agent': array([2., 0.], dtype=float32)}, Reward=-1, Done=False, Truncated=False\n",
      "Step result: Observation={'agent': array([1., 0.], dtype=float32)}, Reward=-0.04, Done=False, Truncated=False\n",
      "Step result: Observation={'agent': array([2., 0.], dtype=float32)}, Reward=-0.04, Done=False, Truncated=False\n"
     ]
    }
   ],
   "source": [
    "# api = GymAPI(\"http://#.##.###.###:####\")\n",
    "# env = MazeEnv(api)\n",
    "\n",
    "# obs, info = env.reset()\n",
    "# print(\"Initial observation:\", obs)\n",
    "\n",
    "# for _ in range(10):\n",
    "#     action = env.action_space.sample() \n",
    "#     obs, reward, done, truncated, info = env.step(action)\n",
    "#     print(f\"Step result: Observation={obs}, Reward={reward}, Done={done}, Truncated={truncated}\")\n",
    "\n",
    "#     if done or truncated:\n",
    "#         break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Έναρξη εκπαίδευσης"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorboard in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (2.18.0)Note: you may need to restart the kernel to use updated packages.\n",
      "\n",
      "Requirement already satisfied: absl-py>=0.4 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard) (2.1.0)\n",
      "Requirement already satisfied: grpcio>=1.48.2 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard) (1.70.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard) (3.7)\n",
      "Requirement already satisfied: numpy>=1.12.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard) (1.26.4)\n",
      "Requirement already satisfied: packaging in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard) (24.2)\n",
      "Requirement already satisfied: protobuf!=4.24.0,>=3.19.6 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard) (5.29.3)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard) (65.5.0)\n",
      "Requirement already satisfied: six>1.9 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard) (1.17.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard) (0.7.2)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tensorboard) (3.1.3)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from werkzeug>=1.0.1->tensorboard) (3.0.2)\n"
     ]
    }
   ],
   "source": [
    "pip install tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from stable_baselines3 import DQN\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "from stable_baselines3.common.monitor import Monitor\n",
    "\n",
    "api = GymAPI(\"http://#.##.###.###:####\")\n",
    "\n",
    "learning_rate = 0.0015\n",
    "exploration_initial_eps = 1.0\n",
    "exploration_final_eps = 0.1\n",
    "exploration_fraction = 0.7\n",
    "learning_starts=1000\n",
    "\n",
    "\n",
    "env = MazeEnv(api)\n",
    "env = Monitor(env, \"./dqn_monitor_logs/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tqdm in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (4.67.1)\n",
      "Requirement already satisfied: rich in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (13.9.4)\n",
      "Requirement already satisfied: colorama in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tqdm) (0.4.6)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from rich) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from rich) (2.19.1)\n",
      "Requirement already satisfied: mdurl~=0.1 in c:\\users\\kprim\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from markdown-it-py>=2.2.0->rich) (0.1.2)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "#pip install tqdm rich"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tqdm \n",
    "import rich"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dqn_model1 = DQN(\"MultiInputPolicy\", env, learning_rate=learning_rate, learning_starts=learning_starts, exploration_initial_eps = exploration_initial_eps, exploration_final_eps = exploration_final_eps, exploration_fraction = exploration_fraction, verbose=1, tensorboard_log=\"./maze_dqn_tensorboard/\")\n",
    "dqn_model1.learn(total_timesteps=100000, tb_log_name=\"dqn_first_run\", progress_bar=True)\n",
    "dqn_model1.save(\"DQN_model1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MER_dqn, STD_dqn = evaluate_policy(dqn_model1, env, n_eval_episodes=100, render=False)\n",
    "print(f\"Mean Reward: {MER_dqn}, Std Reward: {STD_dqn}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PPO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "from stable_baselines3.common.vec_env import SubprocVecEnv\n",
    "#from stable_baselines3.common.vec_env import DummyVecEnv\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "from stable_baselines3.common.monitor import Monitor\n",
    "\n",
    "api = GymAPI(\"http://#.##.###.###:####\")\n",
    "\n",
    "learning_rate = 0.001\n",
    "n_envs = 8\n",
    "n_steps = 1024\n",
    "batch_size = 64\n",
    "n_epochs = 10\n",
    "gamma = 0.9\n",
    "clip_range = 0.1\n",
    "\n",
    "def create_env():\n",
    "    env = MazeEnv(api)\n",
    "    env = Monitor(env, \"./ppo_monitor_logs/\")\n",
    "    return env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n",
      "Logging to ./maze_ppo_tensorboard/ppo_first_run_17\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ace266160eb4734af9fb88f4571f222",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------\n",
      "| rollout/           |          |\n",
      "|    ep_len_mean     | 291      |\n",
      "|    ep_rew_mean     | -115     |\n",
      "| time/              |          |\n",
      "|    fps             | 68       |\n",
      "|    iterations      | 1        |\n",
      "|    time_elapsed    | 119      |\n",
      "|    total_timesteps | 8192     |\n",
      "---------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 356         |\n",
      "|    ep_rew_mean          | -121        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 65          |\n",
      "|    iterations           | 2           |\n",
      "|    time_elapsed         | 251         |\n",
      "|    total_timesteps      | 16384       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.021800369 |\n",
      "|    clip_fraction        | 0.31        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.37       |\n",
      "|    explained_variance   | -0.00676    |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 0.791       |\n",
      "|    n_updates            | 10          |\n",
      "|    policy_gradient_loss | -0.0219     |\n",
      "|    value_loss           | 27.5        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 461         |\n",
      "|    ep_rew_mean          | -137        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 63          |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 385         |\n",
      "|    total_timesteps      | 24576       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.017389048 |\n",
      "|    clip_fraction        | 0.311       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.33       |\n",
      "|    explained_variance   | 0.128       |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 3.58        |\n",
      "|    n_updates            | 20          |\n",
      "|    policy_gradient_loss | -0.0204     |\n",
      "|    value_loss           | 35.6        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 399         |\n",
      "|    ep_rew_mean          | -86.3       |\n",
      "| time/                   |             |\n",
      "|    fps                  | 62          |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 520         |\n",
      "|    total_timesteps      | 32768       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.015070673 |\n",
      "|    clip_fraction        | 0.225       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.26       |\n",
      "|    explained_variance   | 0.152       |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 19.2        |\n",
      "|    n_updates            | 30          |\n",
      "|    policy_gradient_loss | -0.0127     |\n",
      "|    value_loss           | 79.4        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 262         |\n",
      "|    ep_rew_mean          | 9.29        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 62          |\n",
      "|    iterations           | 5           |\n",
      "|    time_elapsed         | 656         |\n",
      "|    total_timesteps      | 40960       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012422793 |\n",
      "|    clip_fraction        | 0.162       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.18       |\n",
      "|    explained_variance   | 0.544       |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 34.3        |\n",
      "|    n_updates            | 40          |\n",
      "|    policy_gradient_loss | -0.0114     |\n",
      "|    value_loss           | 54.9        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 199         |\n",
      "|    ep_rew_mean          | 43.2        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 62          |\n",
      "|    iterations           | 6           |\n",
      "|    time_elapsed         | 792         |\n",
      "|    total_timesteps      | 49152       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012048295 |\n",
      "|    clip_fraction        | 0.15        |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.11       |\n",
      "|    explained_variance   | 0.672       |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 33          |\n",
      "|    n_updates            | 50          |\n",
      "|    policy_gradient_loss | -0.0123     |\n",
      "|    value_loss           | 73.5        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 185        |\n",
      "|    ep_rew_mean          | 54.1       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 61         |\n",
      "|    iterations           | 7          |\n",
      "|    time_elapsed         | 926        |\n",
      "|    total_timesteps      | 57344      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.01216251 |\n",
      "|    clip_fraction        | 0.171      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -1.08      |\n",
      "|    explained_variance   | 0.762      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 17.4       |\n",
      "|    n_updates            | 60         |\n",
      "|    policy_gradient_loss | -0.00986   |\n",
      "|    value_loss           | 45.1       |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 163         |\n",
      "|    ep_rew_mean          | 66.7        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 61          |\n",
      "|    iterations           | 8           |\n",
      "|    time_elapsed         | 1062        |\n",
      "|    total_timesteps      | 65536       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.012512701 |\n",
      "|    clip_fraction        | 0.145       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -1.02       |\n",
      "|    explained_variance   | 0.781       |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 33          |\n",
      "|    n_updates            | 70          |\n",
      "|    policy_gradient_loss | -0.0074     |\n",
      "|    value_loss           | 52.5        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 82.4        |\n",
      "|    ep_rew_mean          | 85.9        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 61          |\n",
      "|    iterations           | 9           |\n",
      "|    time_elapsed         | 1201        |\n",
      "|    total_timesteps      | 73728       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.020704962 |\n",
      "|    clip_fraction        | 0.171       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.964      |\n",
      "|    explained_variance   | 0.823       |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 24.1        |\n",
      "|    n_updates            | 80          |\n",
      "|    policy_gradient_loss | -0.0162     |\n",
      "|    value_loss           | 45.2        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 42.5        |\n",
      "|    ep_rew_mean          | 93.3        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 60          |\n",
      "|    iterations           | 10          |\n",
      "|    time_elapsed         | 1347        |\n",
      "|    total_timesteps      | 81920       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.026578661 |\n",
      "|    clip_fraction        | 0.263       |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.911      |\n",
      "|    explained_variance   | 0.855       |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 43.2        |\n",
      "|    n_updates            | 90          |\n",
      "|    policy_gradient_loss | -0.0358     |\n",
      "|    value_loss           | 67.5        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 20.1       |\n",
      "|    ep_rew_mean          | 98.3       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 59         |\n",
      "|    iterations           | 12         |\n",
      "|    time_elapsed         | 1664       |\n",
      "|    total_timesteps      | 98304      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03378561 |\n",
      "|    clip_fraction        | 0.305      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.605     |\n",
      "|    explained_variance   | 0.907      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 36.2       |\n",
      "|    n_updates            | 110        |\n",
      "|    policy_gradient_loss | -0.0495    |\n",
      "|    value_loss           | 70.6       |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 17.1       |\n",
      "|    ep_rew_mean          | 98.8       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 58         |\n",
      "|    iterations           | 13         |\n",
      "|    time_elapsed         | 1828       |\n",
      "|    total_timesteps      | 106496     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.03692009 |\n",
      "|    clip_fraction        | 0.309      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.387     |\n",
      "|    explained_variance   | 0.927      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 18.4       |\n",
      "|    n_updates            | 120        |\n",
      "|    policy_gradient_loss | -0.049     |\n",
      "|    value_loss           | 47.7       |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 15         |\n",
      "|    ep_rew_mean          | 99.1       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 57         |\n",
      "|    iterations           | 14         |\n",
      "|    time_elapsed         | 2000       |\n",
      "|    total_timesteps      | 114688     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.06359939 |\n",
      "|    clip_fraction        | 0.211      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.175     |\n",
      "|    explained_variance   | 0.953      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 7.61       |\n",
      "|    n_updates            | 130        |\n",
      "|    policy_gradient_loss | -0.0464    |\n",
      "|    value_loss           | 25.3       |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 14.4       |\n",
      "|    ep_rew_mean          | 99.3       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 56         |\n",
      "|    iterations           | 15         |\n",
      "|    time_elapsed         | 2173       |\n",
      "|    total_timesteps      | 122880     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.04408013 |\n",
      "|    clip_fraction        | 0.0659     |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.1       |\n",
      "|    explained_variance   | 0.967      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 5.68       |\n",
      "|    n_updates            | 140        |\n",
      "|    policy_gradient_loss | -0.0211    |\n",
      "|    value_loss           | 15.1       |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 16.4        |\n",
      "|    ep_rew_mean          | 99.2        |\n",
      "| time/                   |             |\n",
      "|    fps                  | 55          |\n",
      "|    iterations           | 16          |\n",
      "|    time_elapsed         | 2341        |\n",
      "|    total_timesteps      | 131072      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.052749343 |\n",
      "|    clip_fraction        | 0.0746      |\n",
      "|    clip_range           | 0.2         |\n",
      "|    entropy_loss         | -0.0674     |\n",
      "|    explained_variance   | 0.99        |\n",
      "|    learning_rate        | 0.001       |\n",
      "|    loss                 | 4.4         |\n",
      "|    n_updates            | 150         |\n",
      "|    policy_gradient_loss | -0.0181     |\n",
      "|    value_loss           | 5.54        |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 15         |\n",
      "|    ep_rew_mean          | 99         |\n",
      "| time/                   |            |\n",
      "|    fps                  | 55         |\n",
      "|    iterations           | 17         |\n",
      "|    time_elapsed         | 2521       |\n",
      "|    total_timesteps      | 139264     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.08811796 |\n",
      "|    clip_fraction        | 0.14       |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.0912    |\n",
      "|    explained_variance   | 0.78       |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 71.2       |\n",
      "|    n_updates            | 160        |\n",
      "|    policy_gradient_loss | -0.0354    |\n",
      "|    value_loss           | 104        |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 15.3       |\n",
      "|    ep_rew_mean          | 98.3       |\n",
      "| time/                   |            |\n",
      "|    fps                  | 54         |\n",
      "|    iterations           | 18         |\n",
      "|    time_elapsed         | 2696       |\n",
      "|    total_timesteps      | 147456     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.07238458 |\n",
      "|    clip_fraction        | 0.172      |\n",
      "|    clip_range           | 0.2        |\n",
      "|    entropy_loss         | -0.0471    |\n",
      "|    explained_variance   | 0.903      |\n",
      "|    learning_rate        | 0.001      |\n",
      "|    loss                 | 13.7       |\n",
      "|    n_updates            | 170        |\n",
      "|    policy_gradient_loss | -0.0461    |\n",
      "|    value_loss           | 46.8       |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Parallel environments\n",
    "vec_env = make_vec_env(create_env, n_envs=n_envs, vec_env_cls=SubprocVecEnv)\n",
    "\n",
    "ppo_model1 = PPO(\"MultiInputPolicy\", vec_env, learning_rate=learning_rate, n_steps=n_steps, gamma=gamma, batch_size=batch_size, n_epochs=n_epochs, verbose=1, tensorboard_log=\"./maze_ppo_tensorboard/\", device = 'cpu')\n",
    "ppo_model1.learn(total_timesteps = 143360, tb_log_name=\"ppo_first_run\", progress_bar=True)\n",
    "ppo_model1.save(\"PPO_model1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Reward: 99.48, Std: 2.842170943040401e-14\n"
     ]
    }
   ],
   "source": [
    "MER_ppo, STD_ppo = evaluate_policy(ppo_model1, vec_env, n_eval_episodes=100)\n",
    "print(f\"Mean Reward: {MER_ppo:.2f}, Std: {STD_ppo}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}